rework LLM section as per mlpack

This reverts commit db5cef9823.

.appveyor.yml

@@ -1,13 +1,9 @@
 environment:
-  ARMADILLO_DOWNLOAD: "http://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz"
+  ARMADILLO_DOWNLOAD: "https://sourceforge.net/projects/arma/files/armadillo-10.8.2.tar.xz"
   BLAS_LIBRARY: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a"
   BLAS_LIBRARY_DLL: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll"
 
   matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-      VSVER: Visual Studio 14 2015 Win64
-      MSBUILD: C:\Program Files (x86)\MSBuild\14.0\bin\MSBuild.exe
-
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       VSVER: Visual Studio 15 2017 Win64
       MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe
@@ -16,6 +12,10 @@ environment:
       VSVER: Visual Studio 16 2019
       MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe
 
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+      VSVER: Visual Studio 17 2022
+      MSBUILD: C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Current\Bin\MSBuild.exe
+
 configuration: Release
 
 install:
@@ -26,13 +26,14 @@ build_script:
   - cd ..
   - appveyor DownloadFile %ARMADILLO_DOWNLOAD% -FileName armadillo.tar.xz
   - 7z x armadillo.tar.xz -so -txz | 7z x -si -ttar > nul
-  - cd armadillo-8.400.0 && mkdir build && cd build
+  - cd armadillo-10.8.2 && mkdir build && cd build
   - >
     cmake -G "%VSVER%"
     -DBLAS_LIBRARY:FILEPATH=%BLAS_LIBRARY%
     -DLAPACK_LIBRARY:FILEPATH=%BLAS_LIBRARY%
     -DCMAKE_PREFIX:FILEPATH="%APPVEYOR_BUILD_FOLDER%/armadillo"
     -DBUILD_SHARED_LIBS=OFF
+    -DCMAKE_POLICY_VERSION_MINIMUM=3.5
     -DCMAKE_BUILD_TYPE=Release ..
   - >
     "%MSBUILD%" "armadillo.sln"
@@ -43,14 +44,15 @@ build_script:
   - cd ensmallen && mkdir build && cd build
   - >
     cmake -G "%VSVER%"
-    -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-8.400.0/include/
+    -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-10.8.2/include/
     -DARMADILLO_LIBRARIES=%BLAS_LIBRARY%
     -DLAPACK_LIBRARY=%BLAS_LIBRARY%
     -DBLAS_LIBRARY=%BLAS_LIBRARY%
+    -DCMAKE_POLICY_VERSION_MINIMUM=3.5
     -DCMAKE_BUILD_TYPE=Release ..
   - >
     "%MSBUILD%" "ensmallen.sln"
-    /m /verbosity:minimal /nologo /p:BuildInParallel=true
+    /m /verbosity:minimal /nologo /p:BuildInParallel=false
 
   # Run tests after copying libraries.
   - ps: cp C:\projects\ensmallen\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* C:\projects\ensmallen\build\

.github/workflows/auto-approve.yml

@@ -0,0 +1,22 @@
+# Once a PR has been approved by one member of the mlpack organization, a second
+# approving review will automatically be added 24 hours later.  This allows time
+# for other maintainers to take a look.
+name: Auto-approve pull requests
+on:
+  schedule:
+   # Run roughly every four hours.
+   - cron: "15 0,4,8,12,16,20 * * *"
+
+jobs:
+  auto-approve:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+     - name: Auto-approve pull requests
+       uses: rcurtin/actions/auto-approve@v1
+       with:
+         repo-token: ${{ secrets.GITHUB_TOKEN }}
+         approval-message:
+           'Second approval provided automatically after 24 hours. :+1:'

.github/workflows/bandicoot-test.yml

@@ -0,0 +1,54 @@
+name: Build and Test
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: ${{ matrix.config.name }}
+    runs-on: ${{ matrix.config.os }}
+    outputs:
+        tag: ${{ steps.git.outputs.tag }}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - { name: 'CUDA', os: self-hosted}
+          - { name: 'OpenCL', os: self-hosted}
+          - { name: 'CPU', os: self-hosted}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Dependencies
+      run: |
+        cd ../
+        rm -rf bandicoot-code
+        git clone --depth 1 --branch 2.1.1 https://gitlab.com/bandicoot-lib/bandicoot-code.git
+        cd bandicoot-code/
+        mkdir build/
+        cd build/
+
+        if [[ "${{ matrix.config.name }}" == "CUDA" ]]; then
+            echo "Installing Bandicoot CUDA"
+            cmake -DFIND_CUDA=ON -DFIND_OPENCL=OFF -DBUILD_TESTS=OFF ../
+            make
+        elif [[ "${{ matrix.config.name }}" == "OpenCL" ]]; then
+            echo "Installing Bandicoot OpenCL"
+            cmake -DFIND_CUDA=OFF -DFIND_OPENCL=ON -DBUILD_TESTS=OFF ../
+            make
+        fi
+
+    - name: Build ensmallen
+      run: |
+          mkdir build
+          cd build/
+
+          if [[ "${{ matrix.config.name }}" == "CPU" ]]; then
+              cmake -DUSE_BANDICOOT=OFF ..
+          else
+              cmake -DBANDICOOT_INCLUDE_DIR=../../bandicoot-code/build/tmp/include/ -DBANDICOOT_LIBRARY=../../bandicoot-code/build/libbandicoot.so ..
+          fi
+
+          make ensmallen_tests
+
+    - name: Test ensmallen
+      run: |
+          cd build/
+          ./ensmallen_tests -d yes

.github/workflows/stale.yml

@@ -0,0 +1,24 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-issues-stale: 30
+          days-before-issue-close: 7
+          stale-issue-label: "s: stale"
+          stale-pr-label: "s: stale"
+          stale-issue-message: "This issue has been automatically marked as stale because it has not had any recent activity.  It will be closed in 7 days if no further activity occurs.  Thank you for your contributions! :+1:"
+          days-before-pr-stale: 30,
+          days-before-pr-close: 14
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          exempt-issue-labels: "s: keep open"
+          exempt-pr-labels: "s: keep open"

.github/workflows/stickers.yml

@@ -0,0 +1,17 @@
+# Post a message to new contributors that they can get some stickers mailed to
+# them.
+name: 'Stickers for new contributors'
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  sticker_comment:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    steps:
+        # Forked version of first-interaction that runs only on first merged PR.
+      - uses: rcurtin/actions/stickers@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          pr-message: "Hello there!  Thanks for your contribution.  Congratulations on your first contribution to mlpack!  If you'd like to add your name to the list of contributors in `COPYRIGHT.txt` and you haven't already, please feel free to push a change to this PR---or, if it gets merged before you can, feel free to open another PR.\n\nIn addition, if you'd like some stickers to put on your laptop, we can get them in the mail for you.  Just send an email with your physical mailing address to stickers@mlpack.org, and then one of the mlpack maintainers will put some stickers in an envelope for you.  It may take a few weeks to get them, depending on your location. :+1:"

.travis.yml

@@ -1,5 +1,5 @@
 os: linux
-dist: trusty
+dist: focal
 language: cpp
 
 env:
@@ -23,10 +23,10 @@ script:
   - if [ $ARMADILLO == "latest" ]; then
       curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.*<a href="\(armadillo-[0-9]*.[0-9]*.[0-9]*.tar.xz\)".*$/\1/' | tail -1` | tar xvJ && cd armadillo*;
     else
-      curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz | tar -xvJ && cd armadillo*;
+      curl -L https://sourceforge.net/projects/arma/files/armadillo-10.8.2.tar.xz | tar -xvJ && cd armadillo*;
     fi
   - cmake . && make && sudo make install && cd ..
-  - mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" .. && make -j2
+  - mkdir build && cd build && cmake .. && make ensmallen_tests -j2
   - CTEST_OUTPUT_ON_FAILURE=1 travis_wait 30 ctest -j2
 
 notifications:

CMake/COOT_FindCLBLAS.cmake

@@ -0,0 +1,44 @@
+# - Find clBLAS (includes and library)
+# This module defines
+#  CLBLAS_INCLUDE_DIR
+#  CLBLAS_LIBRARIES
+#  CLBLAS_FOUND
+# also defined, but not for general use are
+#  CLBLAS_LIBRARY, where to find the library.
+
+find_path(CLBLAS_INCLUDE_DIR clBLAS.h
+/usr/include/
+/usr/local/include/
+)
+
+set(CLBLAS_NAMES ${CLBLAS_NAMES} clBLAS)
+find_library(CLBLAS_LIBRARY
+  NAMES ${CLBLAS_NAMES}
+  PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib
+  )
+
+if (CLBLAS_LIBRARY AND CLBLAS_INCLUDE_DIR)
+    set(CLBLAS_LIBRARIES ${CLBLAS_LIBRARY})
+    set(CLBLAS_FOUND "YES")
+else ()
+  set(CLBLAS_FOUND "NO")
+endif ()
+
+if (CLBLAS_FOUND)
+   if (NOT CLBLAS_FIND_QUIETLY)
+      message(STATUS "Found a clBLAS library: ${CLBLAS_LIBRARIES}")
+   endif ()
+else ()
+    if (CLBLAS_FIND_REQUIRED)
+      message(FATAL_ERROR "Could not find a clBLAS library")
+   endif ()
+endif ()
+
+# Deprecated declarations.
+set (NATIVE_CLBLAS_INCLUDE_PATH ${CLBLAS_INCLUDE_DIR} )
+get_filename_component (NATIVE_CLBLAS_LIB_PATH ${CLBLAS_LIBRARY} PATH)
+
+mark_as_advanced(
+  CLBLAS_LIBRARY
+  CLBLAS_INCLUDE_DIR
+  )

CMake/COOT_FindCLBlast.cmake

@@ -0,0 +1,44 @@
+# - Find clBlast (includes and library)
+# This module defines
+#  CLBLAST_INCLUDE_DIR
+#  CLBLAST_LIBRARIES
+#  CLBLAST_FOUND
+# also defined, but not for general use are
+#  CLBLAST_LIBRARY, where to find the library.
+
+find_path(CLBLAST_INCLUDE_DIR clblast.h
+/usr/include/
+/usr/local/include/
+)
+
+set(CLBLAST_NAMES ${CLBLAST_NAMES} clblast)
+find_library(CLBLAST_LIBRARY
+  NAMES ${CLBLAST_NAMES}
+  PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib
+  )
+
+if (CLBLAST_LIBRARY AND CLBLAST_INCLUDE_DIR)
+    set(CLBLAST_LIBRARIES ${CLBLAST_LIBRARY})
+    set(CLBLAST_FOUND "YES")
+else ()
+  set(CLBLAST_FOUND "NO")
+endif ()
+
+if (CLBLAST_FOUND)
+   if (NOT CLBLAST_FIND_QUIETLY)
+      message(STATUS "Found a clBlast library: ${CLBLAST_LIBRARIES}")
+   endif ()
+else ()
+    if (CLBLAST_FIND_REQUIRED)
+      message(FATAL_ERROR "Could not find a clBlast library")
+   endif ()
+endif ()
+
+# Deprecated declarations.
+set (NATIVE_CLBLAST_INCLUDE_PATH ${CLBLAST_INCLUDE_DIR} )
+get_filename_component (NATIVE_CLBLAST_LIB_PATH ${CLBLAST_LIBRARY} PATH)
+
+mark_as_advanced(
+  CLBLAST_LIBRARY
+  CLBLAST_INCLUDE_DIR
+  )

CMake/COOT_FindNVRTC.cmake

@@ -0,0 +1,35 @@
+# - Find clBlast (includes and library)
+# This module defines
+#  CLBLAST_INCLUDE_DIR
+#  CLBLAST_LIBRARIES
+#  CLBLAST_FOUND
+# also defined, but not for general use are
+#  CLBLAST_LIBRARY, where to find the library.
+
+set(NVRTC_NAMES ${NVRTC_NAMES} nvrtc)
+find_library(NVRTC_LIBRARY
+  NAMES ${NVRTC_NAMES}
+  PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib /usr/lib/x86_64-linux-gnu/
+  )
+
+if (NVRTC_LIBRARY)
+  set(NVRTC_LIBRARIES ${NVRTC_LIBRARY})
+  set(NVRTC_FOUND "YES")
+else ()
+  set(NVRTC_FOUND "NO")
+endif ()
+
+if (NVRTC_FOUND)
+  if (NOT NVRTC_FIND_QUIETLY)
+    message(STATUS "Found NVRTC library: ${NVRTC_LIBRARIES}")
+   endif ()
+else ()
+  if (NVRTC_FIND_REQUIRED)
+    message(FATAL_ERROR "Could not find NVRTC library")
+   endif ()
+endif ()
+
+# Deprecated declarations.
+get_filename_component (NATIVE_NVRTC_LIB_PATH ${NVRTC_LIBRARY} PATH)
+
+mark_as_advanced(NVRTC_LIBRARY)

CMake/FindBandicoot.cmake

@@ -0,0 +1,319 @@
+# - Find Bandicoot
+# Find Bandicoot: GPU accelerator add-on for the Armadillo C++ linear algebra
+# library
+#
+# Using Bandicoot:
+#  find_package(Bandicoot REQUIRED)
+#  include_directories(${BANDICOOT_INCLUDE_DIRS})
+#  add_executable(foo foo.cc)
+#  target_link_libraries(foo ${BANDICOOT_LIBRARIES})
+# This module sets the following variables:
+#  BANDICOOT_FOUND - set to true if the library is found
+#  BANDICOOT_INCLUDE_DIRS - list of required include directories
+#  BANDICOOT_LIBRARIES - list of libraries to be linked
+#  BANDICOOT_VERSION_MAJOR - major version number
+#  BANDICOOT_VERSION_MINOR - minor version number
+#  BANDICOOT_VERSION_PATCH - patch version number
+#  BANDICOOT_VERSION_STRING - version number as a string (ex: "1.0.4")
+#  BANDICOOT_VERSION_NOTE - name of the version (ex: "unstable development version")
+
+find_path(BANDICOOT_INCLUDE_DIR
+  NAMES bandicoot
+  PATHS "$ENV{ProgramFiles}/Bandicoot/include"
+  )
+
+if(BANDICOOT_INCLUDE_DIR)
+  # Extract version information.
+  file(READ "${BANDICOOT_INCLUDE_DIR}/bandicoot_bits/coot_version.hpp" _bandicoot_HEADER_CONTENTS)
+  string(REGEX REPLACE ".*#define COOT_VERSION_MAJOR ([0-9]+).*" "\\1" BANDICOOT_VERSION_MAJOR "${_bandicoot_HEADER_CONTENTS}")
+  string(REGEX REPLACE ".*#define COOT_VERSION_MINOR ([0-9]+).*" "\\1" BANDICOOT_VERSION_MINOR "${_bandicoot_HEADER_CONTENTS}")
+  string(REGEX REPLACE ".*#define COOT_VERSION_PATCH ([0-9]+).*" "\\1" BANDICOOT_VERSION_PATCH "${_bandicoot_HEADER_CONTENTS}")
+  string(REGEX REPLACE ".*#define COOT_VERSION_NOTE\ +\"([0-9a-zA-Z\ _-]+)\".*" "\\1" BANDICOOT_VERSION_NOTE "${_bandicoot_HEADER_CONTENTS}")
+
+  set(BANDICOOT_VERSION_STRING "${BANDICOOT_VERSION_MAJOR}.${BANDICOOT_VERSION_MINOR}.${BANDICOOT_VERSION_PATCH}")
+endif ()
+
+# Determine what support libraries are being used, and whether or not we need to
+# link against them.  We need to look in config.hpp.
+set(SUPPORT_INCLUDE_DIRS "")
+set(SUPPORT_LIBRARIES "")
+set(COOT_NEED_LIBRARY true) # Assume true.
+if(EXISTS "${BANDICOOT_INCLUDE_DIR}/bandicoot_bits/config.hpp")
+  file(READ "${BANDICOOT_INCLUDE_DIR}/bandicoot_bits/config.hpp" _bandicoot_CONFIG_CONTENTS)
+  # COOT_USE_WRAPPER
+  string(REGEX MATCH "\r?\n[\t ]*#define[ \t]+COOT_USE_WRAPPER[ \t]*\r?\n" COOT_USE_WRAPPER "${_bandicoot_CONFIG_CONTENTS}")
+
+  # COOT_USE_OPENCL
+  string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]COOT_USE_OPENCL[)][\t ]*\r?\n[\t
+  ]*#define[ \t]+COOT_USE_OPENCL[ \t]*\r?\n" COOT_USE_OPENCL "${_bandicoot_CONFIG_CONTENTS}")
+
+  # COOT_USE_CUDA
+  string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]COOT_USE_CUDA[)][\t ]*\r?\n[\t
+  ]*#define[ \t]+COOT_USE_CUDA[ \t]*\r?\n" COOT_USE_CUDA "${_bandicoot_CONFIG_CONTENTS}")
+
+  # COOT_USE_LAPACK
+  string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]COOT_USE_LAPACK[)][\t ]*\r?\n[\t ]*#define[ \t]+COOT_USE_LAPACK[ \t]*\r?\n" COOT_USE_LAPACK "${_bandicoot_CONFIG_CONTENTS}")
+
+  # COOT_USE_BLAS
+  string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]COOT_USE_BLAS[)][\t ]*\r?\n[\t ]*#define[ \t]+COOT_USE_BLAS[ \t]*\r?\n" COOT_USE_BLAS "${_bandicoot_CONFIG_CONTENTS}")
+
+  # If we aren't wrapping, things get a little more complex.
+  if(NOT COOT_USE_WRAPPER)
+    set(COOT_NEED_LIBRARY false)
+    message(STATUS "COOT_USE_WRAPPER is not defined, so all dependencies of "
+                   "Bandicoot must be manually linked.")
+
+    set(HAVE_OPENCL false)
+    set(HAVE_CUDA   false)
+    set(HAVE_LAPACK false)
+    set(HAVE_BLAS   false)
+
+    # Search for OpenCL.
+    if (NOT "${COOT_USE_OPENCL}" STREQUAL "" AND NOT HAVE_OPENCL)
+      set(OpenCL_FIND_QUIETLY true)
+      include(FindOpenCL)
+
+      if (OpenCL_FOUND)
+        message(STATUS "OpenCL includes: ${OpenCL_INCLUDE_DIRS}")
+        message(STATUS "OpenCL libraries: ${OpenCL_LIBRARIES}")
+
+        set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+            "${OpenCL_INCLUDE_DIRS}")
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${OpenCL_LIBRARIES}")
+        set(HAVE_OPENCL true)
+      endif ()
+
+      # Search for clBLAS.
+      set(CLBLAS_FIND_QUIETLY true)
+      include(COOT_FindCLBLAS)
+
+      if (CLBLAS_FOUND)
+        message(STATUS "clBLAS includes: ${CLBLAS_INCLUDE_DIR}")
+        message(STATUS "clBLAS libraries: ${CLBLAS_LIBRARIES}")
+
+        set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+            "${CLBLAS_INCLUDE_DIR}")
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${CLBLAS_LIBRARIES}")
+        set(HAVE_CLBLAS true)
+      endif ()
+    endif ()
+
+    # Search for CUDA.
+    if (NOT COOT_USE_CUDA AND NOT HAVE_CUDA)
+      # FindCUDA is deprecated since version 3.10 and replaced with
+      # FindCUDAToolkit wich was added in CMake 3.17.
+      message(STATUS "${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}")
+      if ("${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" LESS "3.67")
+        set(CUDA_FIND_QUIETLY true)
+        find_package(CUDA)
+
+        if (CUDA_FOUND)
+	  message(STATUS "CUDA includes: ${CUDA_INCLUDE_DIRS}")
+	  message(STATUS "CUDA libraries: ${CUDA_LIBRARIES}")
+
+          # We also need NVRTC and also libcuda itself, which the old FindCUDA package do not find.
+          find_library(CUDA_cuda_LIBRARY cuda
+              HINTS ${CUDA_TOOLKIT_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64)
+          find_library(CUDA_nvrtc_LIBRARY nvrtc
+              HINTS ${CUDA_TOOLKIT_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64)
+
+          include(COOT_FindNVRTC)
+
+          if (NVRTC_FOUND)
+            message(STATUS "NVRTC libraries: ${NVRTC_LIBRARIES}")
+            set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${NVRTC_LIBRARIES}")
+          endif ()
+
+          set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+              "${CUDA_INCLUDE_DIRS}")
+          set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}"
+              "${CUDA_LIBRARIES}"
+              "${CUDA_nvrtc_LIBRARY}"
+              "${CUDA_CUDA_LIBRARY}"
+              "${CUDA_CUBLAS_LIBRARIES}"
+              "${CUDA_curand_LIBRARY}"
+              "${CUDA_cusolver_LIBRARY}")
+          set(CUDA_INCLUDE_DIRS "")
+          set(HAVE_CUDA true)
+
+        endif ()
+      else ()
+        set(CUDA_TOOLKIT_FIND_QUIETLY true)
+        find_package(CUDAToolkit REQUIRED)
+
+        if (CUDAToolkit_FOUND)
+	  message(STATUS "CUDA includes: ${CUDAToolkit_INCLUDE_DIRS}")
+	  message(STATUS "CUDA libraries: ${CUDAToolkit_LIBRARY_DIR}")
+
+	  set(CUDA_LIBRARIES CUDA::cudart CUDA::cuda_driver)
+          set(CUDA_CUBLAS_LIBRARIES CUDA::cublas)
+          set(CUDA_curand_LIBRARY CUDA::curand)
+          set(CUDA_cusolver_LIBRARY CUDA::cusolver)
+          set(CUDA_nvrtc_LIBRARY CUDA::nvrtc)
+
+          set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+              "${CUDAToolkit_INCLUDE_DIRS}")
+          set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}"
+              CUDA_LIBRARIES
+	      CUDA_CUBLAS_LIBRARIES
+	      CUDA_curand_LIBRARY
+	      CUDA_cusolver_LIBRARY
+	      CUDA_nvrtc_LIBRARY)
+          set(HAVE_CUDA true)
+        endif()
+      endif ()
+    endif ()
+
+    # Search for LAPACK/BLAS (or replacement).
+    if ((NOT "${COOT_USE_LAPACK}" STREQUAL "") AND
+      (NOT "${COOT_USE_BLAS}" STREQUAL ""))
+      # In order of preference: MKL, ACML, OpenBLAS, ATLAS
+      set(MKL_FIND_QUIETLY true)
+      include(ARMA_FindMKL)
+      set(ACMLMP_FIND_QUIETLY true)
+      include(ARMA_FindACMLMP)
+      set(ACML_FIND_QUIETLY true)
+      include(ARMA_FindACML)
+
+      if (MKL_FOUND)
+        message(STATUS "Using MKL for LAPACK/BLAS: ${MKL_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${MKL_LIBRARIES}")
+        set(HAVE_LAPACK true)
+        set(HAVE_BLAS   true)
+      elseif (ACMLMP_FOUND)
+        message(STATUS "Using multi-core ACML libraries for LAPACK/BLAS:
+            ${ACMLMP_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${ACMLMP_LIBRARIES}")
+        set(HAVE_LAPACK true)
+        set(HAVE_BLAS   true)
+      elseif (ACML_FOUND)
+        message(STATUS "Using ACML for LAPACK/BLAS: ${ACML_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${ACML_LIBRARIES}")
+        set(HAVE_LAPACK true)
+        set(HAVE_BLAS   true)
+      endif ()
+    endif ()
+
+    # If we haven't found BLAS, try.
+    if (NOT "${COOT_USE_BLAS}" STREQUAL "" AND NOT HAVE_BLAS)
+      # Search for BLAS.
+      set(OpenBLAS_FIND_QUIETLY false)
+      include(ARMA_FindOpenBLAS)
+      set(CBLAS_FIND_QUIETLY true)
+      include(ARMA_FindCBLAS)
+      set(BLAS_FIND_QUIETLY true)
+      include(ARMA_FindBLAS)
+
+      if (OpenBLAS_FOUND)
+        # Warn if ATLAS is found also.
+        if (CBLAS_FOUND)
+          message(STATUS "Warning: both OpenBLAS and ATLAS have been found; "
+              "ATLAS will not be used.")
+        endif ()
+        message(STATUS "Using OpenBLAS for BLAS: ${OpenBLAS_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${OpenBLAS_LIBRARIES}")
+        set(HAVE_BLAS true)
+      elseif (CBLAS_FOUND)
+        message(STATUS "Using ATLAS for BLAS: ${CBLAS_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${CBLAS_LIBRARIES}")
+        set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+            "${CBLAS_INCLUDE_DIR}")
+        set(HAVE_BLAS true)
+      elseif (BLAS_FOUND)
+        message(STATUS "Using standard BLAS: ${BLAS_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${BLAS_LIBRARIES}")
+        set(HAVE_BLAS true)
+      endif ()
+    endif ()
+
+    # If we haven't found LAPACK, try.
+    if (NOT "${COOT_USE_LAPACK}" STREQUAL "" AND NOT HAVE_LAPACK)
+      # Search for LAPACK.
+      set(CLAPACK_FIND_QUIETLY true)
+      include(ARMA_FindCLAPACK)
+      set(LAPACK_FIND_QUIETLY true)
+      include(ARMA_FindLAPACK)
+
+      # Only use ATLAS if OpenBLAS isn't being used.
+      if (CLAPACK_FOUND AND NOT OpenBLAS_FOUND)
+        message(STATUS "Using ATLAS for LAPACK: ${CLAPACK_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${CLAPACK_LIBRARIES}")
+        set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+            "${CLAPACK_INCLUDE_DIR}")
+        set(HAVE_LAPACK true)
+      elseif (LAPACK_FOUND)
+        message(STATUS "Using standard LAPACK: ${LAPACK_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${LAPACK_LIBRARIES}")
+        set(HAVE_LAPACK true)
+      endif ()
+    endif ()
+
+    if (NOT "${COOT_USE_LAPACK}" STREQUAL "" AND NOT HAVE_LAPACK)
+      message(FATAL_ERROR "Cannot find LAPACK library, but COOT_USE_LAPACK is "
+          "set. Try specifying LAPACK libraries manually by setting the "
+          "LAPACK_LIBRARY variable.")
+    endif ()
+
+    if (NOT "${COOT_USE_BLAS}" STREQUAL "" AND NOT HAVE_BLAS)
+      message(FATAL_ERROR "Cannot find BLAS library, but COOT_USE_BLAS is set. "
+          "Try specifying BLAS libraries manually by setting the BLAS_LIBRARY "
+          "variable.")
+    endif ()
+
+  endif()
+else()
+  message(STATUS "${BANDICOOT_INCLUDE_DIR}/bandicoot_bits/config.hpp not "
+      "found!  Cannot determine what to link against.")
+endif()
+
+if (COOT_NEED_LIBRARY)
+  # UNIX paths are standard, no need to write.
+  find_library(BANDICOOT_LIBRARY
+    NAMES bandicoot
+    PATHS "$ENV{ProgramFiles}/Bandicoot/lib"  "$ENV{ProgramFiles}/Bandicoot/lib64" "$ENV{ProgramFiles}/Bandicoot"
+    )
+
+  # Checks 'REQUIRED', 'QUIET' and versions.
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(Bandicoot
+    REQUIRED_VARS BANDICOOT_LIBRARY BANDICOOT_INCLUDE_DIR
+    VERSION_VAR BANDICOOT_VERSION_STRING)
+else ()
+  # Checks 'REQUIRED', 'QUIET' and versions.
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(Bandicoot
+    REQUIRED_VARS BANDICOOT_INCLUDE_DIR
+    VERSION_VAR BANDICOOT_VERSION_STRING)
+endif ()
+
+if (BANDICOOT_FOUND)
+  # Also include support include directories.
+  set(BANDICOOT_INCLUDE_DIRS ${BANDICOOT_INCLUDE_DIR} ${SUPPORT_INCLUDE_DIRS})
+  # Also include support libraries to link against.
+  if (COOT_NEED_LIBRARY)
+    set(BANDICOOT_LIBRARIES ${BANDICOOT_LIBRARY} ${SUPPORT_LIBRARIES})
+  else ()
+    set(BANDICOOT_LIBRARIES ${SUPPORT_LIBRARIES})
+  endif ()
+  message(STATUS "Bandicoot libraries: ${BANDICOOT_LIBRARIES}")
+  message(STATUS "Bandicoot includes: ${BANDICOOT_INCLUDE_DIR}")
+endif ()
+
+# Hide internal variables
+mark_as_advanced(
+  BANDICOOT_INCLUDE_DIR
+  BANDICOOT_LIBRARIES)
+
+if (BANDICOOT_FOUND AND NOT TARGET Bandicoot::Bandicoot)
+  add_library(Bandicoot::Bandicoot INTERFACE IMPORTED)
+  set_target_properties(Bandicoot::Bandicoot PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${BANDICOOT_INCLUDE_DIR}"
+      INTERFACE_LINK_LIBRARIES "${BANDICOOT_LIBRARIES}")
+endif()

CMake/FindCBLAS.cmake

@@ -0,0 +1,75 @@
+
+# ==================================================================================================
+# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
+# project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
+# width of 100 characters per line.
+#
+# Author(s):
+#   Cedric Nugteren <www.cedricnugteren.nl>
+#
+# ==================================================================================================
+#
+# Defines the following variables:
+#   CBLAS_FOUND          Boolean holding whether or not the Netlib BLAS library was found
+#   CBLAS_INCLUDE_DIRS   The Netlib BLAS include directory
+#   CBLAS_LIBRARIES      The Netlib BLAS library
+#
+# In case BLAS is not installed in the default directory, set the CBLAS_ROOT variable to point to
+# the root of BLAS, such that 'cblas.h' can be found in $CBLAS_ROOT/include. This can either be
+# done using an environmental variable (e.g. export CBLAS_ROOT=/path/to/BLAS) or using a CMake
+# variable (e.g. cmake -DCBLAS_ROOT=/path/to/BLAS ..).
+#
+# ==================================================================================================
+
+# Sets the possible install locations
+set(CBLAS_HINTS
+  ${CBLAS_ROOT}
+  $ENV{CBLAS_ROOT}
+)
+set(CBLAS_PATHS
+  /usr
+  /usr/local
+  /usr/local/opt
+  /System/Library/Frameworks
+)
+
+# Finds the include directories
+find_path(CBLAS_INCLUDE_DIRS
+  NAMES cblas.h
+  HINTS ${CBLAS_HINTS}
+  PATH_SUFFIXES
+    include inc include/x86_64 include/x64
+    openblas/include include/blis blis/include blis/include/blis
+    Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current/Headers
+  PATHS ${CBLAS_PATHS}
+  DOC "Netlib BLAS include header cblas.h"
+)
+mark_as_advanced(CBLAS_INCLUDE_DIRS)
+
+# Finds the library
+find_library(CBLAS_LIBRARIES
+  NAMES cblas blas blis openblas accelerate
+  HINTS ${CBLAS_HINTS}
+  PATH_SUFFIXES
+    lib lib64 lib/x86_64 lib/x64 lib/x86 lib/Win32 lib/import lib64/import
+    openblas/lib blis/lib lib/atlas-base
+  PATHS ${CBLAS_PATHS}
+  DOC "Netlib BLAS library"
+)
+mark_as_advanced(CBLAS_LIBRARIES)
+
+# ==================================================================================================
+
+# Notification messages
+if(NOT CBLAS_INCLUDE_DIRS)
+    message(STATUS "Could NOT find 'cblas.h', install a CPU Netlib BLAS or set CBLAS_ROOT")
+endif()
+if(NOT CBLAS_LIBRARIES)
+    message(STATUS "Could NOT find a CPU Netlib BLAS library, install it or set CBLAS_ROOT")
+endif()
+
+# Determines whether or not BLAS was found
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(CBLAS DEFAULT_MSG CBLAS_INCLUDE_DIRS CBLAS_LIBRARIES)
+
+# ==================================================================================================

CMakeLists.txt

@@ -1,19 +1,26 @@
 # ensmallen CMake configuration.  This project has no configurable options---it
 # just installs the headers to the install location, and optionally builds the
 # test program.
-cmake_minimum_required(VERSION 3.3.2)
-project(ensmallen 
+cmake_minimum_required(VERSION 3.5...4.0)
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE "Release")  # ensure the tests are built with optimisation
+endif ()
+project(ensmallen
         LANGUAGES C CXX)
 
 # Configurable options for CMake.
 option(USE_OPENMP "If available, use OpenMP for parallelization." ON)
-option(BUILD_TESTS "Build tests." ON)
+option(USE_BANDICOOT "If available, build against Bandicoot for GPU support." ON)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake")
 
-# Set required C++ standard to C++11.
-set(CMAKE_CXX_STANDARD 11)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
+# Set minimum required C++ standard to C++14.
+if (NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 14)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+elseif (${CMAKE_CXX_STANDARD} LESS 14)
+  message(FATAL_ERROR "ensmallen requires C++14 or newer!")
+endif ()
 
 # Extract version from sources.
 set(ENSMALLEN_VERSION_FILE_NAME "${PROJECT_SOURCE_DIR}/include/ensmallen_bits/ens_version.hpp")
@@ -40,7 +47,7 @@ target_include_directories(ensmallen INTERFACE
 if(MSVC)
   target_compile_options(ensmallen INTERFACE $<BUILD_INTERFACE:/Wall>)
 else()
-  target_compile_options(ensmallen INTERFACE $<BUILD_INTERFACE:-Wall -Wpedantic -Wunused-parameter>)
+  target_compile_options(ensmallen INTERFACE $<BUILD_INTERFACE:-Wall -Wpedantic -Wunused>)
 endif()
 
 # Find OpenMP and link it.
@@ -57,8 +64,19 @@ if(USE_OPENMP)
   target_link_libraries(ensmallen INTERFACE OpenMP::OpenMP_CXX)
 endif()
 
+if(USE_BANDICOOT)
+  # Find Bandicoot and link it.
+  find_package(Bandicoot 2.1.0)
+  if(BANDICOOT_FOUND)
+    target_link_libraries(ensmallen INTERFACE Bandicoot::Bandicoot)
+    target_include_directories(ensmallen INTERFACE ${BANDICOOT_INCLUDE_DIR})
+
+    add_definitions(-DENS_USE_COOT)
+  endif()
+endif()
+
 # Find Armadillo and link it.
-find_package(Armadillo 8.400.0 REQUIRED)
+find_package(Armadillo 10.8.2 REQUIRED)
 target_link_libraries(ensmallen INTERFACE Armadillo::Armadillo)
 
 # Set helper variables for creating the version, config and target files.
@@ -96,6 +114,4 @@ install(FILES ${CMAKE_SOURCE_DIR}/include/ensmallen.hpp
 
 # Enable testing and build tests.
 enable_testing()
-if (BUILD_TESTS)
-  add_subdirectory(tests)
-endif()
+add_subdirectory(tests)

CONTRIBUTING.md

@@ -8,9 +8,27 @@ you have an improvement you would like to see, we would love to include it!
 
 The ensmallen maintainer community overlaps heavily with the
 [mlpack](https://github.com/mlpack/mlpack) community, so development discussions
-can happen either here on Github, on the [mlpack mailing
-list](http://lists.mlpack.org/mailman/listinfo/mlpack), or in the #mlpack
-IRC channel on irc.freenode.net.
+can happen either here on Github or in the `#mlpack:matrix.org` channel on
+[Matrix](https://www.matrix.org/).  See
+[here](https://www.mlpack.org/doc/developer/community.html) for more
+information.
+
+## Usage of LLMs / Coding Assistants
+
+Each contributor needs to understand the content they are proposing for ensmallen,
+and how the proposed content changes and/or expands functionality in ensmallen.
+Content includes source code, documentation, and other material such as images
+and datasets.  The pull request must justify the proposed content in the
+description.  Using coding assistants or tools like Large Language Models (LLMs) 
+does not grant additional privileges or reduce our expectations.
+
+Each contributor is responsible for the proposed content, regardless of where
+the content came from.  The responsibility includes ensuring that the content
+can be submitted to ensmallen and does not violate intellectual property rights
+such as copyright(s).  Source code in ensmallen is licensed under the BSD license;
+see `LICENSE.txt` for details.
+
+## Pull request process
 
 Once a pull request is submitted, it must be reviewed and approved before a
 merge, to ensure that:
@@ -108,8 +126,8 @@ $ cd ensmallen
 
 # - or -
 
-$ wget http://ensmallen.org/files/ensmallen-2.16.2.tar.gz
-$ tar -xvzpf ensmallen-2.16.2.tar.gz
+$ wget http://ensmallen.org/files/ensmallen-3.11.0.tar.gz
+$ tar -xvzpf ensmallen-3.11.0.tar.gz
 $ cd ensmallen-latest
 ```
 

COPYRIGHT.txt

@@ -7,10 +7,10 @@ Source:
 
 Files: *
 Copyright:
-  Copyright 2008-2018, Ryan Curtin <ryan@ratml.org>
+  Copyright 2008-2023, Ryan Curtin <ryan@ratml.org>
   Copyright 2008-2012, Dongryeol Lee <dongryel@cc.gatech.edu>
   Copyright 2010-2012, James Cline <james.cline@gatech.edu>
-  Copyright 2013-2018, Marcus Edel <marcus.edel@fu-berlin.de>
+  Copyright 2013-2023, Marcus Edel <marcus.edel@fu-berlin.de>
   Copyright 2013-2018, Sumedh Ghaisas <sumedhghaisas@gmail.com>
   Copyright 2013, Mudit Raj Gupta <mudit.raaj.gupta@gmail.com>
   Copyright 2014, Ryan Birmingham <birm@gatech.edu>
@@ -33,7 +33,7 @@ Copyright:
   Copyright 2018, B Kartheek Reddy <bkartheekreddy@gmail.com>
   Copyright 2018, Moksh Jain <mokshjn00@gmail.com>
   Copyright 2018, Shikhar Jaiswal <jaiswalshikhar87@gmail.com>
-  Copyright 2018, Conrad Sanderson
+  Copyright 2018-2023, Conrad Sanderson
   Copyright 2018, Dan Timson
   Copyright 2019, Rahul Ganesh Prabhu
   Copyright 2019, Roberto Hueso <robertohueso96@gmail.com>

HISTORY.md

@@ -1,3 +1,251 @@
+### ensmallen ?.??.?: "???"
+###### ????-??-??
+
+### ensmallen 3.11.0: "Sunny Day"
+###### 2025-12-15
+ * Refactor `GradientDescent` into
+   `GradientDescentType<UpdatePolicyType, DecayPolicyType>` and
+   add the `DeltaBarDelta` and `MomentumDeltaBarDelta` optimizers
+   ([#440](https://github.com/mlpack/ensmallen/pull/440)).
+
+ * Fix an off-by-one bug where the actual number of executed iterations was one
+   fewer than the specified `maxIterations`
+   ([#443](https://github.com/mlpack/ensmallen/pull/443)).
+
+### ensmallen 3.10.0: "Unexpected Rain"
+###### 2025-09-25
+ * SGD-like optimizers now all divide the step size by the batch size so that
+   step sizes don't need to be tuned in addition to batch sizes.  If you require
+   behavior from ensmallen 2, define the `ENS_OLD_SEPARABLE_STEP_BEHAVIOR` macro
+   before including `ensmallen.hpp`
+   ([#431](https://github.com/mlpack/ensmallen/pull/431)).
+
+ * Remove deprecated `ParetoFront()` and `ParetoSet()` from multi-objective
+   optimizers ([#435](https://github.com/mlpack/ensmallen/pull/435)).  Instead,
+   pass objects to the `Optimize()` function; see the documentation for each
+   multi-objective optimizer for more details.  A typical transition will change
+   code like:
+
+    ```c++
+    optimizer.Optimize(objectives, coordinates);
+    arma::cube paretoFront = optimizer.ParetoFront();
+    arma::cube paretoSet = optimizer.ParetoSet();
+    ```
+
+   to instead gather the Pareto front and set in the call:
+
+    ```c++
+    arma::cube paretoFront, paretoSet;
+    optimizer.Optimize(objectives, coordinates, paretoFront, paretoSet);
+    ```
+
+ * Remove deprecated constructor for Active CMA-ES that takes `lowerBound` and
+   `upperBound` ([#435](https://github.com/mlpack/ensmallen/pull/435)).
+   Instead, pass an instantiated `BoundaryBoxConstraint` to the constructor.  A
+   typical transition will change code like:
+
+    ```c++
+    ActiveCMAES<FullSelection, BoundaryBoxConstraint> opt(lambda,
+        lowerBound, upperBound, ...);
+    ```
+
+   into
+
+    ```c++
+    ActiveCMAES<FullSelection, BoundaryBoxConstraint> opt(lambda,
+        BoundaryBoxConstraint(lowerBound, upperBound), ...);
+    ```
+
+ * Add proximal gradient optimizers for L1-constrained and other related
+   problems: `FBS`, `FISTA`, and `FASTA`
+   ([#427](https://github.com/mlpack/ensmallen/pull/427)).  See the
+   documentation for more details.
+
+ * The `Lambda()` and `Sigma()` functions of the `AugLagrangian` optimizer,
+   which could be used to retrieve the Lagrange multipliers and penalty
+   parameter after optimization, are now deprecated
+   ([#439](https://github.com/mlpack/ensmallen/pull/439)).  Instead, pass a
+   vector and a double to the `Optimize()` function directly:
+
+    ```c++
+    augLag.Optimize(function, coordinates, lambda, sigma)
+    ```
+
+   and these will be filled with the final Lagrange multiplier estimates and
+   penalty parameters.
+
+### ensmallen 2.22.2: "E-Bike Excitement"
+###### 2025-04-30
+ * Fix include statement in `tests/de_test.cpp`
+   ([#419](https://github.com/mlpack/ensmallen/pull/419)).
+
+ * Fix `exactObjective` output for SGD-like optimizers when the number of
+   iterations is an even number of epochs
+   ([#417](https://github.com/mlpack/ensmallen/pull/417)).
+
+ * Increase tolerance in `demon_sgd_test.cpp`
+   ([#420](https://github.com/mlpack/ensmallen/pull/420)).
+
+ * Set cmake version range to 3.5...4.0
+   ([#422](https://github.com/mlpack/ensmallen/pull/422)).
+
+### ensmallen 2.22.1: "E-Bike Excitement"
+###### 2024-12-02
+ * Remove unused variables to fix compiler warnings
+   ([#413](https://github.com/mlpack/ensmallen/pull/413)).
+
+### ensmallen 2.22.0: "E-Bike Excitement"
+###### 2024-11-29
+ * Update to C++14 standard
+   ([#400](https://github.com/mlpack/ensmallen/pull/400)).
+
+ * Bump minimum Armadillo version to 10.8
+   ([#404](https://github.com/mlpack/ensmallen/pull/404)).
+
+ * For Armadillo 14.2.0 switch to `.index_min()` and `.index_max()`
+   ([#409](https://github.com/mlpack/ensmallen/pull/409)).
+
+ * Added IPOP and BIPOP restart mechanisms for CMA-ES.
+   ([#403](https://github.com/mlpack/ensmallen/pull/403)).
+
+
+### ensmallen 2.21.1: "Bent Antenna"
+###### 2024-02-15
+ * Fix numerical precision issues for small-gradient L-BFGS scaling factor
+   computations ([#392](https://github.com/mlpack/ensmallen/pull/392)).
+
+ * Ensure the tests are built with optimisation enabled
+   ([#394](https://github.com/mlpack/ensmallen/pull/394)).
+
+### ensmallen 2.21.0: "Bent Antenna"
+###### 2023-11-27
+ * Clarify return values for different callback types
+   ([#383](https://github.com/mlpack/ensmallen/pull/383)).
+
+ * Fix return types of callbacks
+   ([#382](https://github.com/mlpack/ensmallen/pull/382)).
+
+ * Minor cleanup for printing optimization reports via `Report()`
+   ([#385](https://github.com/mlpack/ensmallen/pull/385)).
+
+### ensmallen 2.20.0: "Stripped Bolt Head"
+###### 2023-10-02
+ * Implementation of Active CMAES
+   ([#367](https://github.com/mlpack/ensmallen/pull/367)).
+
+ * LBFGS: avoid generation of NaNs, and add checks for finite values
+   ([#368](https://github.com/mlpack/ensmallen/pull/368)).
+
+ * Fix CNE test tolerances
+   ([#360](https://github.com/mlpack/ensmallen/pull/360)).
+
+ * Rename `SCD` optimizer, to `CD`
+   ([#379](https://github.com/mlpack/ensmallen/pull/379)).
+
+### ensmallen 2.19.1: "Eight Ball Deluxe"
+###### 2023-01-30
+ * Avoid deprecation warnings in Armadillo 11.2+
+   ([#347](https://github.com/mlpack/ensmallen/pull/347)).
+
+### ensmallen 2.19.0: "Eight Ball Deluxe"
+###### 2022-04-06
+ * Added DemonSGD and DemonAdam optimizers
+   ([#211](https://github.com/mlpack/ensmallen/pull/211)).
+
+ * Fix bug with Adam-like optimizers not resetting when `resetPolicy` is `true`.
+   ([#340](https://github.com/mlpack/ensmallen/pull/340)).
+
+ * Add Yogi optimizer
+   ([#232](https://github.com/mlpack/ensmallen/pull/232)).
+
+ * Add AdaBelief optimizer
+   ([#233](https://github.com/mlpack/ensmallen/pull/233)).
+
+ * Add AdaSqrt optimizer
+   ([#234](https://github.com/mlpack/ensmallen/pull/234)).
+
+ * Bump check for minimum supported version of Armadillo
+   ([#342](https://github.com/mlpack/ensmallen/pull/342)).
+
+### ensmallen 2.18.2: "Fairmount Bagel"
+###### 2022-02-13
+ * Update Catch2 to 2.13.8
+  ([#336](https://github.com/mlpack/ensmallen/pull/336)).
+
+ * Fix epoch timing output
+  ([#337](https://github.com/mlpack/ensmallen/pull/337)).
+
+### ensmallen 2.18.1: "Fairmount Bagel"
+###### 2021-11-19
+ * Accelerate SGD test time
+   ([#330](https://github.com/mlpack/ensmallen/pull/300)).
+
+ * Fix potential infinite loop in CMAES
+   ([#331](https://github.com/mlpack/ensmallen/pull/331)).
+
+ * Fix SCD partial gradient test
+   ([#332](https://github.com/mlpack/ensmallen/pull/332)).
+
+### ensmallen 2.18.0: "Fairmount Bagel"
+###### 2021-10-20
+ * Add gradient value clipping and gradient norm scaling callback
+   ([#315](https://github.com/mlpack/ensmallen/pull/315)).
+
+ * Remove superfluous CMake option to build the tests
+   ([#313](https://github.com/mlpack/ensmallen/pull/313)).
+
+ * Bump minimum Armadillo version to 9.800
+   ([#318](https://github.com/mlpack/ensmallen/pull/318)).
+
+ * Update Catch2 to 2.13.7
+   ([#322](https://github.com/mlpack/ensmallen/pull/322)).
+
+ * Remove redundant template argument for C++20 compatibility
+   ([#324](https://github.com/mlpack/ensmallen/pull/324)).
+
+ * Fix MOEAD test stability
+   ([#327](https://github.com/mlpack/ensmallen/pull/327)).
+
+### ensmallen 2.17.0: "Pachis Din Me Pesa Double"
+###### 2021-07-06
+ * CheckArbitraryFunctionTypeAPI extended for MOO support
+   ([#283](https://github.com/mlpack/ensmallen/pull/283)).
+
+ * Refactor NSGA2
+   ([#263](https://github.com/mlpack/ensmallen/pull/263),
+   [#304](https://github.com/mlpack/ensmallen/pull/304)).
+
+ * Add Indicators for Multiobjective optimizers
+   ([#285](https://github.com/mlpack/ensmallen/pull/285)).
+
+ * Make Callback flexible for MultiObjective Optimizers
+   ([#289](https://github.com/mlpack/ensmallen/pull/289)).
+
+ * Add ZDT Test Suite
+   ([#273](https://github.com/mlpack/ensmallen/pull/273)).
+
+ * Add MOEA-D/DE Optimizer
+   ([#269](https://github.com/mlpack/ensmallen/pull/269)).
+
+ * Introduce Policy Methods for MOEA/D-DE
+   ([#293](https://github.com/mlpack/ensmallen/pull/293)).
+
+ * Add Das-Dennis weight initialization method
+   ([#295](https://github.com/mlpack/ensmallen/pull/295)).
+
+ * Add Dirichlet Weight Initialization
+   ([#296](https://github.com/mlpack/ensmallen/pull/296)).
+
+ * Improved installation and compilation instructions
+   ([#300](https://github.com/mlpack/ensmallen/pull/300)).
+
+ * Disable building the tests by default for faster installation
+   ([#303](https://github.com/mlpack/ensmallen/pull/303)).
+
+ * Modify matrix initialisation to take into account
+   default element zeroing in Armadillo 10.5
+   ([#305](https://github.com/mlpack/ensmallen/pull/305)).
+
 ### ensmallen 2.16.2: "Severely Dented Can Of Polyurethane"
 ###### 2021-03-24
  * Fix CNE test trials

README.md

@@ -2,33 +2,35 @@
   <a href="http://ensmallen.org/"><img src="http://ensmallen.org/img/ensmallen_text.svg" style="background-color:rgba(0,0,0,0);" height=230 alt="ensmallen: a C++ header-only library for numerical optimization"></a>
 </h2>
 
-**ensmallen** is a C++ header-only library for numerical optimization.
+**ensmallen** is a high-quality C++ library for non-linear numerical optimization.
 
-Documentation and downloads: http://ensmallen.org
-
-ensmallen provides a simple set of abstractions for writing an objective
-function to optimize. It also provides a large set of standard and cutting-edge
-optimizers that can be used for virtually any numerical optimization task.
-These include full-batch gradient descent techniques, small-batch techniques,
-gradient-free optimizers, and constrained optimization.
+ensmallen provides many types of optimizers that can be used
+for virtually any numerical optimization task.
+This includes gradient descent techniques, gradient-free optimizers,
+and constrained optimization.
+Examples include L-BFGS, SGD, CMAES and Simulated Annealing.
+ensmallen also allows optional callbacks to customize the optimization process.
 
+Documentation and downloads: https://ensmallen.org
 
 ### Requirements
 
-* C++ compiler with C++11 support
-* Armadillo: http://arma.sourceforge.net
+* C++ compiler with C++14 support
+* Armadillo (https://arma.sourceforge.net), version 10.8.2 or later
 * OpenBLAS or Intel MKL or LAPACK (see Armadillo site for details)
 
 
 ### Installation
 
-ensmallen can be installed with CMake 3.3 or later.
-If CMake is not already available on your system, it can be obtained from https://cmake.org
+ensmallen can be installed in several ways: either manually or via cmake,
+with or without root access.
 
-If you are using an older system such as RHEL 7 or CentOS 7,
-an updated version of CMake is also available via the EPEL repository via the `cmake3` package.
+The cmake based installation will check the requirements
+and optionally build the tests. If cmake 3.3 (or a later version)
+is not already available on your system, it can be obtained
+from [cmake.org](https://cmake.org).
 
-Example installation:
+Example cmake based installation with root access:
 
 ```
 mkdir build
@@ -37,10 +39,54 @@ cmake ..
 sudo make install
 ```
 
+Example cmake based installation without root access,
+installing into `/home/blah/` (adapt as required):
 
-### Example Usage
+```
+mkdir build
+cd build
+cmake .. -DCMAKE_INSTALL_PREFIX:PATH=/home/blah/
+make install
+```
 
-See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer in a linear regression setting.
+The above will create a directory named `/home/blah/include/`
+and place all ensmallen headers there.
+
+To optionally build and run the tests
+(after running cmake as above),
+use the following additional commands:
+
+```
+make ensmallen_tests
+./ensmallen_tests --durations yes
+```
+
+Manual installation involves simply copying the `include/ensmallen.hpp` header
+***and*** the associated `include/ensmallen_bits` directory to a location
+such as `/usr/include/` which is searched by your C++ compiler.
+If you can't use `sudo` or don't have write access to `/usr/include/`,
+use a directory within your own home directory (eg. `/home/blah/include/`).
+
+
+### Example Compilation
+
+If you have installed ensmallen in a standard location such as `/usr/include/`:
+
+    g++ prog.cpp -o prog -O2 -larmadillo
+
+If you have installed ensmallen in a non-standard location,
+such as `/home/blah/include/`, you will need to make sure
+that your C++ compiler searches `/home/blah/include/`
+by explicitly specifying the directory as an argument/option.
+For example, using the `-I` switch in gcc and clang:
+
+    g++ prog.cpp -o prog -O2 -I /home/blah/include/ -larmadillo
+
+
+### Example Optimization
+
+See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer
+in a linear regression setting.
 
 
 ### License
@@ -48,7 +94,7 @@ See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer in a
 Unless stated otherwise, the source code for **ensmallen** is licensed under the
 3-clause BSD license (the "License").  A copy of the License is included in the
 "LICENSE.txt" file.  You may also obtain a copy of the License at
-http://opensource.org/licenses/BSD-3-Clause .
+http://opensource.org/licenses/BSD-3-Clause
 
 
 ### Citation
@@ -57,28 +103,20 @@ Please cite the following paper if you use ensmallen in your research and/or
 software. Citations are useful for the continued development and maintenance of
 the library.
 
-* S. Bhardwaj, R. Curtin, M. Edel, Y. Mentekidis, C. Sanderson.  
-  [ensmallen: a flexible C++ library for efficient function optimization](http://www.ensmallen.org/files/ensmallen_2018.pdf).  
-  Workshop on Systems for ML and Open Source Software at NIPS 2018.
+* Ryan R. Curtin, Marcus Edel, Rahul Ganesh Prabhu, Suryoday Basak, Zhihao Lou, Conrad Sanderson.
+  [The ensmallen library for flexible numerical optimization](https://jmlr.org/papers/volume22/20-416/20-416.pdf).
+  Journal of Machine Learning Research, Vol. 22, No. 166, 2021.
 
 ```
-@article{DBLP:journals/corr/abs-1810-09361,
-  author    = {Shikhar Bhardwaj and
-               Ryan R. Curtin and
-               Marcus Edel and
-               Yannis Mentekidis and
-               Conrad Sanderson},
-  title     = {ensmallen: a flexible {C++} library for efficient function optimization},
-  journal   = {CoRR},
-  volume    = {abs/1810.09361},
-  doi       = {10.5281/zenodo.2008650},
-  year      = {2018},
-  url       = {http://arxiv.org/abs/1810.09361},
-  archivePrefix = {arXiv},
-  eprint    = {1810.09361},
-  timestamp = {Wed, 31 Oct 2018 14:24:29 +0100},
-  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1810-09361},
-  bibsource = {dblp computer science bibliography, https://dblp.org}
+@article{ensmallen_JMLR_2021,
+  author  = {Ryan R. Curtin and Marcus Edel and Rahul Ganesh Prabhu and Suryoday Basak and Zhihao Lou and Conrad Sanderson},
+  title   = {The ensmallen library for flexible numerical optimization},
+  journal = {Journal of Machine Learning Research},
+  year    = {2021},
+  volume  = {22},
+  number  = {166},
+  pages   = {1--6},
+  url     = {http://jmlr.org/papers/v22/20-416.html}
 }
 ```
 
@@ -111,3 +149,4 @@ the library.
 * N Rajiv Vaidyanathan
 * Roberto Hueso
 * Sayan Goswami
+

doc/callbacks.md

@@ -1,10 +1,10 @@
-Callbacks in ensmallen are methods that are called at various states during the
-optimization process, which can be used to implement and control behaviors such
-as:
+Callbacks in ensmallen are methods that are called at various stages of the
+optimization process.  These can be used to print information about the optimization, modify behavior of the optimization, or a wide range of other possibilities.  Some examples of what callbacks can be used for include:
 
 * Changing the learning rate.
-* Printing of the current objective.
-* Sending a message when the optimization hits a specific state such us a minimal objective.
+* Printing the current objective.
+* Sending a message when the optimization hits a specific state such us a
+  minimal objective.
 
 Callbacks can be passed as an argument to the `Optimize()` function:
 
@@ -73,6 +73,27 @@ std::cout << callback.BestObjective() << std::endl;
 
 </details>
 
+Numerous implemented and ready-to-use callbacks are included with ensmallen, and
+it is also easy to write a custom callback.
+
+ * [`EarlyStopAtMinLoss`](#earlystopatminloss): stop the optimization if no
+   improvement has been made
+ * [`GradClipByNorm`](#gradclipbynorm): reduce the norm of the gradient to
+   prevent the exploding gradient problem
+ * [`GradClipByValue`](#gradclipbyvalue): clip the gradient to specified minimum
+   and maximum values
+ * [`PrintLoss`](#printloss): print the objective at each iteration to a
+   specified stream
+ * [`ProgressBar`](#progressbar): print a progress bar to the screen at each
+   iteration
+ * [`Report`](#report): print a report at the end of optimization
+ * [`StoreBestCoordinates`](#storebestcoordinates): store the coordinates that
+   give the best objective value at the end of an epoch
+ * [`TimerStop`](#timerstop): stop the optimization after a given amount of time
+
+A [guide for implementing custom callbacks](#custom-callbacks) is below, and a
+few [example custom callbacks](#custom-callback-examples) are given too.
+
 ## Built-in Callbacks
 
 ### EarlyStopAtMinLoss
@@ -136,7 +157,7 @@ EarlyStopAtMinLoss cb(
       // You could also, e.g., print the validation loss here to watch it converge.
       return lrfValidation.Evaluate(coordinates);
     });
-    
+
 arma::mat coordinates = lrfTrain.GetInitialPoint();
 SMORMS3 smorms3;
 smorms3.Optimize(lrfTrain, coordinates, cb);
@@ -144,6 +165,82 @@ smorms3.Optimize(lrfTrain, coordinates, cb);
 
 </details>
 
+### Gradient Clipping
+
+One challenge in optimization is dealing with "exploding gradients", where large
+parameter gradients can cause the optimizer to make excessively large updates,
+potentially pushing the model into regions of high loss or causing numerical
+instability. This can happen due to:
+
+* A high learning rate, leading to large gradient updates.
+* Poorly scaled datasets, resulting in significant variance between data points.
+* A loss function that generates disproportionately large error values.
+
+Common solutions for this problem are:
+
+#### GradClipByNorm
+
+In this method, the solution is to change the derivative
+of the error before applying the update step.  One option is to clip the norm
+`||g||` of the gradient `g` before a parameter update. So given the gradient,
+and a maximum norm value, the callback normalizes the gradient so that its
+L2-norm is less than or equal to the given maximum norm value.
+
+#### Constructors
+
+ * `GradClipByNorm(`_`maxNorm`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`maxNorm`** | The maximum clipping value. | |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, GradClipByNorm(0.3));
+```
+
+#### GradClipByValue
+
+In this method, the solution is to change the derivative
+of the error before applying the update step.  One option is to clip the
+parameter gradient element-wise before a parameter update.
+
+#### Constructors
+
+ * `GradClipByValue(`_`min, max`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`min`** | The minimum value to clip to. | |
+| `double` | **`max`** | The maximum value to clip to. | |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, GradClipByValue(0, 1.3));
+```
+
 ### PrintLoss
 
 Callback that prints loss to stdout or a specified output stream.
@@ -208,7 +305,7 @@ optimizer.Optimize(f, coordinates, ProgressBar());
 
 </details>
 
-### Report 
+### Report
 
 Callback that prints a optimizer report to stdout or a specified output stream.
 
@@ -336,143 +433,304 @@ std::cout << "The optimized model found by AdaDelta has the "
 
 </details>
 
-## Callback States
+### TimerStop
 
-Callbacks are called at several states during the optimization process:
+Callback that stops optimization after a certain amount of time has elapsed.
 
-* At the beginning and end of the optimization process.
-* After any call to `Evaluate()` and `EvaluateConstraint`.
-* After any call to `Gradient()` and `GradientConstraint`.
-* At the start and end of an epoch.
+#### Constructors
 
-Each callback provides optimization relevant information that can be accessed or
-modified.
+ * `TimerStop(`_`seconds`_`)`
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+// Limit optimization to 15 seconds.
+optimizer.Optimize(f, coordinates, TimerStop(15));
+```
+
+</details>
+
+## Custom Callbacks
+
+Custom callbacks can be easily implemented by creating a class and simply
+implementing functions for each individual callback that you are interested in
+handling, plus any other functionality you might need (e.g. constructors,
+accessors).
+
+Thus, when writing a custom callback, start with an empty class like this:
+
+```c++
+class CustomCallback
+{
+ public:
+  // Add individual callback handlers that you are interested in handling!
+};
+```
+
+and add any of the individual callback handler functions described below.
 
 ### BeginOptimization
 
-Called at the beginning of the optimization process.
+Called at the beginning of the optimization process.  Add this function to your
+callback class with your desired implementation:
 
- * `BeginOptimization(`_`optimizer, function, coordinates`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  void BeginOptimization(OptimizerType& optimizer,
+                         FunctionType& function,
+                         MatType& coordinates);
+```
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function to be optimized (e.g. the first argument given to
+   `optimizer.Optimize()`.
+ * `coordinates`: the current coordinates for optimization; since optimization
+   is just beginning, this is the exact same matrix given to `Optimize()`.
 
 ### EndOptimization
 
-Called at the end of the optimization process.
+Called at the end of the optimization process.  Add this function to your
+callback class with your desired implementation:
 
- * `EndOptimization(`_`optimizer, function, coordinates`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  void EndOptimization(OptimizerType& optimizer,
+                       FunctionType& function,
+                       MatType& coordinates);
+```
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that has been optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the final coordinates for optimization; since optimization is
+   ending, these are the same values that will be in the resulting matrix after
+   `Optimize()` finishes.
 
 ### Evaluate
 
-Called after any call to `Evaluate()`.
+Called after any call to `Evaluate()` or `EvaluateWithGradient()`.  Add this
+function to your callback class with your desired implementation:
 
- * `Evaluate(`_`optimizer, function, coordinates, objective`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool Evaluate(OptimizerType& optimizer,
+                FunctionType& function,
+                const MatType& coordinates,
+                const double objective);
+```
 
-#### Attributes
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates with which `function.Evaluate()` was called.
+ * `objective`: the result of `function.Evaluate(coordinates)`.
 
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `double` | **`objective`** | Objective value of the current point. |
+If the callback returns `true`, the optimization will be terminated.
 
 ### EvaluateConstraint
 
- Called after any call to `EvaluateConstraint()`.
+Called after any call to `EvaluateConstraint()`, for
+[constrained functions](#constrained-functions).  Add this function to your
+callback class with your desired implementation:
 
- * `EvaluateConstraint(`_`optimizer, function, coordinates, constraint, constraintValue`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool EvaluateConstraint(OptimizerType& optimizer,
+                          FunctionType& function,
+                          const MatType& coordinates,
+                          const size_t constraintIndex,
+                          const double constraintValue);
+```
 
-#### Attributes
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates with which `function.EvaluateConstraint()` was
+   called.
+ * `constraintIndex`: the index of the constraint that was evaluated
+ * `constraintValue`: the result of
+   `function.EvaluateConstraint(coordinates, constraintIndex)`.
 
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `size_t` | **`constraint`** | The index of the constraint. |
-| `double` | **`constraintValue`** | Constraint value of the current point. |
+If the callback returns `true`, the optimization will be terminated.
 
 ### Gradient
 
- Called after any call to `Gradient()`.
+Called after any call to `Gradient()` or `EvaluateWithGradient()`.  Add this
+function to your callback class with your desired implementation:
 
- * `Gradient(`_`optimizer, function, coordinates, gradient`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  bool Gradient(OptimizerType& optimizer,
+                FunctionType& function,
+                const MatType& coordinates,
+                GradType& gradient);
+```
 
-#### Attributes
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates with which `function.Gradient()` was called.
+ * `gradient`: the computed gradient (can be modified!).
 
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `GradType` | **`gradient`** | Matrix that holds the gradient. |
+If the callback returns `true`, the optimization will be terminated.
 
 ### GradientConstraint
 
- Called after any call to `GradientConstraint()`.
+Called after any call to `GradientConstraint()` for
+[constrained functions](#constrained-functions).  Add this function to your
+callback class with your desired implementation:
 
- * `GradientConstraint(`_`optimizer, function, coordinates, constraint, gradient`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  bool GradientConstraint(OptimizerType& optimizer,
+                          FunctionType& function,
+                          const MatType& coordinates,
+                          const size_t constraintIndex,
+                          GradType& constraintGradient);
+```
 
-#### Attributes
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates with which `function.GradientConstraint()` was
+   called.
+ * `constraintIndex`: the index of the constraint whose gradient was computed.
+ * `constraintGradient`: the computed result of
+   `function.GradientConstraint()`.
 
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `size_t` | **`constraint`** | The index of the constraint. |
-| `GradType` | **`gradient`** | Matrix that holds the gradient. |
+If the callback returns `true`, the optimization will be terminated.
 
 ### BeginEpoch
 
-Called at the beginning of a pass over the data. The objective may be exact or
-an estimate depending on `exactObjective` value.
+Called at the beginning of a pass over the data, for
+[separable functions](#separable-functions). The objective may be exact or
+an estimate depending on the optimizer's `ExactObjective()` value.  Add this
+function to your callback class with your desired implementation:
 
- * `BeginEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool BeginEpoch(OptimizerType& optimizer,
+                  FunctionType& function,
+                  const MatType& coordinates,
+                  const size_t epoch,
+                  const double objective);
+```
 
-#### Attributes
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates at the start of the epoch.
+ * `epoch`: the epoch number.
+ * `objective`: the exact or approximate objective at the end of the previous
+   epoch.
 
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `size_t` | **`epoch`** | The index of the current epoch. |
-| `double` | **`objective`** | Objective value of the current point. |
+If the callback returns `true`, the optimization will be terminated.
 
 ### EndEpoch
 
-Called at the end of a pass over the data. The objective may be exact or
-an estimate depending on `exactObjective` value.
+Called at the end of a pass over the data, for
+[separable functions](#separable-functions). The objective may be exact or an
+estimate depending on the optimizer's `ExactObjective()` value.  Add this
+function to your callback class with your desired implementation:
 
- * `EndEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool EndEpoch(OptimizerType& optimizer,
+                FunctionType& function,
+                const MatType& coordinates,
+                const size_t epoch,
+                const double objective);
+```
 
-#### Attributes
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates at the end of the epoch.
+ * `epoch`: the epoch number.
+ * `objective`: the exact or approximate objective at the end of the epoch.
 
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `size_t` | **`epoch`** | The index of the current epoch. |
-| `double` | **`objective`** | Objective value of the current point. |
+If the callback returns `true`, the optimization will be terminated.
 
-## Custom Callbacks
+### StepTaken
+
+Called after the optimizer has taken any step that modifies the coordinates.
+Add this function to your callback class with your desired implementation:
+
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool StepTaken(OptimizerType& optimizer,
+                FunctionType& function,
+                MatType& coordinates);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates after the step (can be modified!).
+
+If the callback returns `true`, the optimization will be terminated.  Note that
+changing the `coordinates` matrix may cause strange behavior for certain
+optimizers---your mileage may vary!
+
+### GenerationalStepTaken
+
+Called after the evolution of a single generation, for
+[multi-objective functions](#multi-objective-functions).  Add this function to
+your callback class with your desired implementation:
+
+```c++
+  template<typename OptimizerType,
+           typename... FunctionTypes,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  bool GenerationalStepTaken(OptimizerType& optimizer,
+                             std::tuple<FunctionTypes...>& functions,
+                             MatType& coordinates,
+                             ObjectivesVecType& objectives,
+                             IndicesType& frontIndices);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `functions`: the functions that are being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates after taking the step.
+ * `objectives`: a vector of column vectors indicating the objective for each
+   element in the population on each objective function.
+ * `frontIndices`: indices of the population that are on the Pareto front.
+
+If the callback returns `true`, the optimization will be terminated.
+
+## Custom Callback Examples
 
 ### Learning rate scheduling
 
@@ -515,7 +773,7 @@ class ExponentialDecay
   // Callback function called at the end of a pass over the data. We are only
   // interested in the current epoch and the optimizer, we ignore the rest.
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& optimizer,
+  bool EndEpoch(OptimizerType& optimizer,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */,
                 const size_t epoch,
@@ -524,6 +782,9 @@ class ExponentialDecay
     // Update the learning rate.
     optimizer.StepSize() = learningRate * (1.0 - std::pow(decay,
         (double) epoch));
+
+    // Do not terminate the optimization.
+    return false;
   }
 
   double learningRate;
@@ -595,7 +856,7 @@ class EarlyStop
   // the current objective. We are only interested in the objective and ignore
   // the rest.
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& /* optimizer */,
+  bool EndEpoch(OptimizerType& /* optimizer */,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */,
                 const size_t /* epoch */,

doc/function_types.md

@@ -130,8 +130,13 @@ Each of the implemented methods is allowed to have additional cv-modifiers
 The following optimizers can be used with differentiable functions:
 
  * [L-BFGS](#l-bfgs) (`ens::L_BFGS`)
+ * [Forward-backward splitting (FBS)](#forward-backward-splitting-fbs) (`ens::FBS`)
+ * [Fast Iterative Shrinkage-Thresholding Algorithm (FISTA)](#fast-iterative-shrinkage-thresholding-algorithm-fista) (`ens::FISTA`)
+ * [Fast Adaptive Shrinkage/Thresholding Algorithm (FASTA)](#fast-adaptive-shrinkage-thresholding-algorithm-fasta) (`ens::FASTA`)
  * [FrankWolfe](#frank-wolfe) (`ens::FrankWolfe`)
  * [GradientDescent](#gradient-descent) (`ens::GradientDescent`)
+ * [DeltaBarDelta](#deltabardelta) (`ens::DeltaBarDelta`)
+ * [MomentumDeltaBarDelta](#momentum-deltabardelta) (`ens::MomentumDeltaBarDelta`)
  - Any optimizer for [arbitrary functions](#arbitrary-functions)
 
 Each of these optimizers has an `Optimize()` function that is called as
@@ -210,6 +215,12 @@ class LinearRegressionEWGFunction
     g = -2 * data * v;
     return arma::accu(v % v); // equivalent to \| v \|^2
   }
+
+ private:
+  // The data.
+  const arma::mat& data;
+  // The responses to each data point.
+  const arma::rowvec& responses;
 };
 
 int main()
@@ -249,7 +260,7 @@ int main()
   const double time1 = clock.toc();
 
   std::cout << "LinearRegressionFunction with Evaluate() and Gradient() took "
-    << time1 << " seconds to converge to the model: " << std::endl;
+      << time1 << " seconds to converge to the model: " << std::endl;
   std::cout << lrf1Params.t();
 
   // Create the second objective function, which uses EvaluateWithGradient().
@@ -273,6 +284,150 @@ int main()
 
 </details>
 
+### Proximal operators and non-differentiable functions
+
+Some optimization problems involve non-differentiable components, and can be
+expressed as the optimization below:
+
+$$ \operatorname{argmin}_x h(x) = \operatorname{argmin}_x f(x) + g(x). $$
+
+Here, `f(x)` is a regular differentiable function (as in the previous
+subsection), and `g(x)` is a non-differentiable arbitrary function.  These
+classes of functions can still be optimized using *proximal gradient
+optimizers*.  The following proximal gradient optimizers are implemented in
+ensmallen (these can also optimize differentiable functions only, taking `g(x) =
+0`):
+
+ * [Forward-backward splitting (FBS)](#forward-backward-splitting-fbs) (`ens::FBS`)
+ * [Fast Iterative Shrinkage-Thresholding Algorithm (FISTA)](#fast-iteartive-shrinkage-thresholding-algorithm-fista) (`ens::FISTA`)
+ * [Fast Adaptive Shrinkage/Thresholding Algorithm (FASTA)](#fast-adaptive-shrinkage-thresholding-algorithm-fasta) (`ens::FASTA`)
+
+ensmallen implements a few `g(x)` options that can be used with proximal
+gradient optimizers:
+
+ * `L1Penalty(`_`lambda`_`)`: $g(x) = \lambda \| x \|_1$
+ * `L1Constraint(`_`lambda`_`)`: $g(x)$ is the constraint $\| x \|_1 \le \lambda$
+
+For example, by pairing `L1Penalty` with the `LinearRegressionFunction` from the
+previous section, we can implement L1-penalized (sparse) linear regression:
+
+<details>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+#include <ensmallen.hpp>
+
+// Define a differentiable objective function by implementing only
+// EvaluateWithGradient().
+class LinearRegressionEWGFunction
+{
+ public:
+  // Construct the object with the given data matrix and responses.
+  LinearRegressionEWGFunction(const arma::mat& dataIn,
+                              const arma::rowvec& responsesIn) :
+      data(dataIn), responses(responsesIn) { }
+
+  // Simultaneously compute both the objective function and gradient for model
+  // parameters x.  Note that this is faster than implementing Evaluate() and
+  // Gradient() individually because it caches the computation of
+  // (responses - x.t() * data)!
+  double EvaluateWithGradient(const arma::mat& x, arma::mat& g)
+  {
+    const arma::rowvec v = (responses - x.t() * data);
+    g = -2 * data * v;
+    return arma::accu(v % v); // equivalent to \| v \|^2
+  }
+
+ private:
+  // The data.
+  const arma::mat& data;
+  // The responses to each data point.
+  const arma::rowvec& responses;
+};
+
+int main()
+{
+  // First, generate some random data, with 1000 points and 500 dimensions.
+  // This data has no pattern and as such will make a model that's not very
+  // useful---but the purpose here is just demonstration. :)
+  //
+  // For a more "real world" situation, load a dataset from file using X.load()
+  // and y.load() (but make sure the matrix is column-major, so that each
+  // observation/data point corresponds to a *column*, *not* a row.
+  arma::mat data(500, 1000, arma::fill::randn);
+  arma::rowvec responses(1000, arma::fill::randn);
+
+  // Create a starting point for our optimization as the vector of all zeros.
+  // The model has 500 parameters, so the shape is 500x1.
+  arma::mat startingPoint(500, 1, arma::fill::zeros);
+
+  // Construct the objective function f(x), and the penalty function g(x) with a
+  // lambda value of 0.1.
+  LinearRegressionEWGFunction lrf(data, responses);
+  ens::L1Penalty g(0.1);
+
+  // Create the FBS optimizer with default parameters, and optimize the function
+  // f(x) + g(x) (i.e. L1-penalized linear regression).
+  // The ens::FBS class can be replaced with any ensmallen proximal gradient
+  // optimizer.
+  ens::FBS fbs(g);
+  arma::mat lrfParams(startingPoint);
+  fbs.Optimize(lrf, lrfParams);
+
+  // Count the number of nonzeros in the final model.
+  // To get fewer nonzeros, the penalty value (0.1) could be increased.
+  std::cout << "Number of nonzeros in optimized parameter vector: "
+      << arma::accu(lrfParams != 0) << "." << std::endl;
+}
+```
+
+</details>
+
+It is possible to implement a custom proximal operator (`g(x)`).  To do so, a
+class with two methods (`Evaluate()` and `BackwardStep()`) must be defined:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+// Compute the value of g(x).
+double Evaluate(const arma::mat& x);
+
+// Perform a backward step (proximal step) on `x`, given that the forward step
+// size was `stepSize`.
+void BackwardStep(arma::mat& x, const double stepSize);
+```
+
+</details>
+
+A simple implementation is below for the `L1Penalty` class:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+double L1Penalty::Evaluate(const arma::mat& coordinates) const
+{
+  // Compute the L1 penalty.
+  return norm(vectorise(coordinates), 1) * lambda;
+}
+
+void L1Penalty::ProximalStep(arma::mat& coordinates,
+                             const double stepSize) const
+{
+  // Apply the backwards step coordinate-wise.
+  // (See Goldstein, Studer, and Baraniuk 2009, eq. (12).)
+  coordinates.transform([this, stepSize](double val) { return (val > 0.0) ?
+      (std::max(0.0, val - lambda * stepSize)) :
+      (std::min(0.0, val + lambda * stepSize)); });
+}
+```
+
+</details>
+
 ### Partially differentiable functions
 
 Some differentiable functions have the additional property that the gradient
@@ -307,7 +462,7 @@ regular implementation of the `Gradient()`, so that function may be omitted.
 If these functions are implemented, the following partially differentiable
 function optimizers can be used:
 
- - [Stochastic Coordinate Descent](#stochastic-coordinate-descent-scd)
+ - [Coordinate Descent](#coordinate-descent-cd)
 
 ## Arbitrary separable functions
 
@@ -358,7 +513,10 @@ Each of the implemented methods is allowed to have additional cv-modifiers
 
 The following optimizers can be used with arbitrary separable functions:
 
- - [CMAES](#cmaes)
+ - [Active CMA-ES](#active-cma-es)
+ - [BIPOP CMA-ES](#bipop-cma-es)
+ - [CMA-ES](#cma-es)
+ - [IPOP CMA-ES](#ipop-cma-es)
 
 Each of these optimizers has an `Optimize()` function that is called as
 `Optimize(f, x)` where `f` is the function to be optimized and `x` holds the
@@ -453,7 +611,7 @@ int main()
   // parameters, so the shape is 10x1.
   arma::mat params(10, 1, arma::fill::randn);
 
-  // Use the CMAES optimizer with default parameters to minimize the
+  // Use the CMA-ES optimizer with default parameters to minimize the
   // LinearRegressionFunction.
   // The ens::CMAES type can be replaced with any suitable ensmallen optimizer
   // that can handle arbitrary separable functions.
@@ -461,7 +619,7 @@ int main()
   LinearRegressionFunction lrf(data, responses);
   cmaes.Optimize(lrf, params);
 
-  std::cout << "The optimized linear regression model found by CMAES has the "
+  std::cout << "The optimized linear regression model found by CMA-ES has the "
       << "parameters " << params.t();
 }
 ```
@@ -554,9 +712,11 @@ Each of the implemented methods is allowed to have additional cv-modifiers
 
 The following optimizers can be used with differentiable separable functions:
 
+ - [AdaBelief](#adabelief)
  - [AdaBound](#adabound)
  - [AdaDelta](#adadelta)
  - [AdaGrad](#adagrad)
+ - [AdaSqrt](#adasqrt)
  - [Adam](#adam)
  - [AdaMax](#adamax)
  - [AMSBound](#amsbound)
@@ -875,19 +1035,110 @@ arma::mat coordinates(6, 1, arma::fill::randu);
 NSGA2 nsga;
 double bestFrontSum = nsga.Optimize(objectives, coordinates);
 
-// Set `bestFront` to contain all of the coordinates on the best front.
-std::vector<arma::mat> bestFront = optimizer.Front();
-}
+// If the entire Pareto front is desired, pass it to the Optimize() function:
+arma::cube front, paretoSet;
+double bestFrontSum2 = nsga.Optimize(objectives, coordinates, front, paretoSet);
 ```
 
 </details>
 
-*Note*: all multi-objective function optimizers have both the function `Optimize()` to find the
-best front, and also the function `Front()` to return all sets of coordinates that are on the
-front.
+### Performance Indicators
+
+Performance indicators in multiobjective optimization provide essential metrics 
+for evaluating solution quality, such as convergence to the Pareto front and solution 
+diversity.
+
+The ensmallen library offers three such indicators, aiding in the assessment and comparison 
+of different optimization methods:
+
+#### Epsilon
+
+Epsilon metric is a performance metric used in multi-objective optimization which measures 
+the smallest factor by which a set of solution objectives must be scaled to dominate a 
+reference set of solutions. Specifically, given a set of Pareto-optimal solutions, the 
+epsilon indicator finds the minimum value ϵ such that each solution in the set is at 
+least as good as every solution in the reference set when the objectives are scaled by ϵ.
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+arma::cube referenceFront(2, 1, 3);
+double tol = 1e-10;
+referenceFront.slice(0) = arma::vec{0.01010101, 0.89949622};
+referenceFront.slice(1) = arma::vec{0.02020202, 0.85786619};
+referenceFront.slice(2) = arma::vec{0.03030303, 0.82592234};
+arma::cube front = referenceFront * 1.1;
+// eps is approximately 1.1
+double eps = Epsilon::Evaluate(front, referenceFront);
+```
+</details>
+
+#### IGD
+
+Inverse Generational Distance (IGD) is a performance metric used in multi-objective optimization 
+to evaluate the quality of a set of solutions relative to a reference set, typically representing 
+the true Pareto front. IGD measures the average distance from each point in the reference set to 
+the closest point in the obtained solution set.
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+arma::cube referenceFront(2, 1, 3);
+double tol = 1e-10;
+referenceFront.slice(0) = arma::vec{0.01010101, 0.89949622};
+referenceFront.slice(1) = arma::vec{0.02020202, 0.85786619};
+referenceFront.slice(2) = arma::vec{0.03030303, 0.82592234};
+arma::cube front = referenceFront * 1.1;
+// The third parameter is the power constant in the distance formula.
+// IGD is approximately 0.05329
+double igd = IGD::Evaluate(front, referenceFront, 1);
+```
+</details>
+
+#### IGD Plus
+
+IGD Plus (IGD+) is a variant of the Inverse Generational Distance (IGD) metric used in multi-objective 
+optimization. It refines the traditional IGD metric by incorporating a preference for Pareto-dominance 
+in the distance calculation. This modification helps IGD+ better reflect both the convergence to the 
+Pareto front and the diversity of the solution set.
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+arma::cube referenceFront(2, 1, 3);
+double tol = 1e-10;
+referenceFront.slice(0) = arma::vec{0.01010101, 0.89949622};
+referenceFront.slice(1) = arma::vec{0.02020202, 0.85786619};
+referenceFront.slice(2) = arma::vec{0.03030303, 0.82592234};
+arma::cube front = referenceFront * 1.1;
+// IGDPlus is approximately 0.05329
+double igdPlus = IGDPlus::Evaluate(front, referenceFront);
+```
+</details>
+
+*Note*: all multi-objective function optimizers have two versions of
+`Optimize()`: one that finds only the best front, and one that also allows
+passing `arma::cube`s (or similar) to return the Pareto set (e.g. the Pareto
+optimal points in variable space) as well as the entire Pareto front (e.g. all
+sets of solutions on the front):
+
+ * `Optimize(`_`functions`_`,`_`coordinates`_`)`
+ * `Optimize(`_`functions`_`,`_`coordinates`_`,`_`paretoSet`_`,`_`paretoFront`_`)`
 
 The following optimizers can be used with multi-objective functions:
 - [NSGA2](#nsga2)
+- [MOEA/D-DE](#moead)
+- [AGEMOEA](#agemoea)
+
+#### See also:
+* [Performance Assessment of Multiobjective Optimizers: An Analysis and Review](https://sop.tik.ee.ethz.ch/publicationListFiles/ztlf2003a.pdf)
+* [Modified Distance Calculation in Generational Distance and Inverted Generational Distance](https://link.springer.com/chapter/10.1007/978-3-319-15892-1_8)
 
 ## Constrained functions
 

include/ensmallen.hpp

@@ -15,22 +15,36 @@
 #ifndef ENSMALLEN_HPP
 #define ENSMALLEN_HPP
 
-// certain compilers are way behind the curve
-#if (defined(_MSVC_LANG) && (_MSVC_LANG >= 201402L))
-  #undef  ARMA_USE_CXX11
-  #define ARMA_USE_CXX11
+#undef ENS_HAVE_CXX14
+
+#if (__cplusplus >= 201402L)
+  #define ENS_HAVE_CXX14
+#endif
+
+#if defined(_MSVC_LANG)
+  #if (_MSVC_LANG >= 201402L)
+    #undef  ENS_HAVE_CXX14
+    #define ENS_HAVE_CXX14
+  #endif
+#endif
+
+#if !defined(ENS_HAVE_CXX14)
+  #error "*** C++14 compiler required; enable C++14 mode in your compiler, or use an earlier version of ensmallen"
 #endif
 
 #include <armadillo>
 
-#if !defined(ARMA_USE_CXX11)
-  // armadillo automatically enables ARMA_USE_CXX11
-  // when a C++11/C++14/C++17/etc compiler is detected
-  #error "please enable C++11/C++14 mode in your compiler"
+#if defined(COOT_VERSION_MAJOR) && \
+    ((COOT_VERSION_MAJOR >= 2) || \
+     (COOT_VERSION_MAJOR == 2 && COOT_VERSION_MINOR >= 1))
+  // The version of Bandicoot is new enough that we can use it.
+  #undef ENS_HAVE_COOT
+  #define ENS_HAVE_COOT
 #endif
 
-#if ((ARMA_VERSION_MAJOR < 8) || ((ARMA_VERSION_MAJOR == 8) && (ARMA_VERSION_MINOR < 400)))
-  #error "need Armadillo version 8.400 or later"
+#if ((ARMA_VERSION_MAJOR < 10) || \
+    ((ARMA_VERSION_MAJOR == 10) && (ARMA_VERSION_MINOR < 8)))
+  #error "need Armadillo version 10.8 or newer"
 #endif
 
 #include <cctype>
@@ -64,7 +78,13 @@
 #include "ensmallen_bits/log.hpp" // TODO: should move to another place
 
 #include "ensmallen_bits/utility/any.hpp"
-#include "ensmallen_bits/utility/arma_traits.hpp"
+#include "ensmallen_bits/utility/proxies.hpp"
+#include "ensmallen_bits/utility/function_traits.hpp"
+#include "ensmallen_bits/utility/using.hpp"
+#include "ensmallen_bits/utility/detect_callbacks.hpp"
+#include "ensmallen_bits/utility/indicators/epsilon.hpp"
+#include "ensmallen_bits/utility/indicators/igd.hpp"
+#include "ensmallen_bits/utility/indicators/igd_plus.hpp"
 
 // Contains traits, must be placed before report callback.
 #include "ensmallen_bits/function.hpp" // TODO: should move to function/
@@ -72,27 +92,41 @@
 // Callbacks.
 #include "ensmallen_bits/callbacks/callbacks.hpp"
 #include "ensmallen_bits/callbacks/early_stop_at_min_loss.hpp"
+#include "ensmallen_bits/callbacks/grad_clip_by_norm.hpp"
+#include "ensmallen_bits/callbacks/grad_clip_by_value.hpp"
 #include "ensmallen_bits/callbacks/print_loss.hpp"
 #include "ensmallen_bits/callbacks/progress_bar.hpp"
+#include "ensmallen_bits/callbacks/query_front.hpp"
 #include "ensmallen_bits/callbacks/report.hpp"
 #include "ensmallen_bits/callbacks/store_best_coordinates.hpp"
 #include "ensmallen_bits/callbacks/timer_stop.hpp"
 
 #include "ensmallen_bits/problems/problems.hpp" // TODO: should move to another place
 
+#include "ensmallen_bits/ada_belief/ada_belief.hpp"
 #include "ensmallen_bits/ada_bound/ada_bound.hpp"
 #include "ensmallen_bits/ada_delta/ada_delta.hpp"
 #include "ensmallen_bits/ada_grad/ada_grad.hpp"
+#include "ensmallen_bits/ada_sqrt/ada_sqrt.hpp"
 #include "ensmallen_bits/adam/adam.hpp"
+#include "ensmallen_bits/demon_adam/demon_adam.hpp"
+#include "ensmallen_bits/demon_sgd/demon_sgd.hpp"
 #include "ensmallen_bits/qhadam/qhadam.hpp"
 #include "ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp"
 #include "ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp"
 #include "ensmallen_bits/cmaes/cmaes.hpp"
+#include "ensmallen_bits/cmaes/active_cmaes.hpp"
+#include "ensmallen_bits/cmaes/pop_cmaes.hpp"
+#include "ensmallen_bits/cd/cd.hpp"
 #include "ensmallen_bits/cne/cne.hpp"
 #include "ensmallen_bits/de/de.hpp"
+#include "ensmallen_bits/delta_bar_delta/delta_bar_delta.hpp"
+#include "ensmallen_bits/delta_bar_delta/momentum_delta_bar_delta.hpp"
 #include "ensmallen_bits/eve/eve.hpp"
+#include "ensmallen_bits/fasta/fasta.hpp"
+#include "ensmallen_bits/fbs/fbs.hpp"
+#include "ensmallen_bits/fista/fista.hpp"
 #include "ensmallen_bits/ftml/ftml.hpp"
-
 #include "ensmallen_bits/fw/frank_wolfe.hpp"
 #include "ensmallen_bits/gradient_descent/gradient_descent.hpp"
 #include "ensmallen_bits/grid_search/grid_search.hpp"
@@ -100,6 +134,8 @@
 #include "ensmallen_bits/katyusha/katyusha.hpp"
 #include "ensmallen_bits/lbfgs/lbfgs.hpp"
 #include "ensmallen_bits/lookahead/lookahead.hpp"
+#include "ensmallen_bits/agemoea/agemoea.hpp"
+#include "ensmallen_bits/moead/moead.hpp"
 #include "ensmallen_bits/nsga2/nsga2.hpp"
 #include "ensmallen_bits/padam/padam.hpp"
 #include "ensmallen_bits/parallel_sgd/parallel_sgd.hpp"
@@ -108,7 +144,6 @@
 
 #include "ensmallen_bits/sa/sa.hpp"
 #include "ensmallen_bits/sarah/sarah.hpp"
-#include "ensmallen_bits/scd/scd.hpp"
 #include "ensmallen_bits/sdp/sdp.hpp"
 #include "ensmallen_bits/sdp/lrsdp.hpp"
 #include "ensmallen_bits/sdp/primal_dual.hpp"
@@ -125,5 +160,6 @@
 #include "ensmallen_bits/svrg/svrg.hpp"
 #include "ensmallen_bits/swats/swats.hpp"
 #include "ensmallen_bits/wn_grad/wn_grad.hpp"
+#include "ensmallen_bits/yogi/yogi.hpp"
 
 #endif

include/ensmallen_bits/ada_belief/ada_belief.hpp

@@ -0,0 +1,186 @@
+/**
+ * @file ada_belief.hpp
+ * @author Marcus Edel
+ *
+ * Class wrapper for the AdaBelief update Policy. The intuition for AdaBelief is
+ * to adapt the stepsize according to the "belief" in the current gradient
+ * direction.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_HPP
+#define ENSMALLEN_ADA_BELIEF_HPP
+
+#include <ensmallen_bits/sgd/sgd.hpp>
+#include "ada_belief_update.hpp"
+
+namespace ens {
+
+/**
+ * The intuition for AdaBelief is to adapt the stepsize according to the
+ * "belief" in the current gradient direction. For more information, see the
+ * following.
+ *
+ * @code
+ * @misc{zhuang2020adabelief,
+ *   title         = {AdaBelief Optimizer: Adapting Stepsizes by the Belief in
+ *                    Observed Gradients},
+ *   author        = {Juntang Zhuang and Tommy Tang and Sekhar Tatikonda and
+ *                    Nicha Dvornek and Yifan Ding and Xenophon Papademetris
+ *                    and James S. Duncan},
+ *   year          = {2020},
+ *   eprint        = {2010.07468},
+ *   archivePrefix = {arXiv},
+ * }
+ * @endcode
+ *
+ * AdaBelief can optimize differentiable separable functions. For more details,
+ * see the documentation on function types included with this distribution or
+ * on the ensmallen website.
+ */
+class AdaBelief
+{
+ public:
+  /**
+   * Construct the AdaBelief optimizer with the given function and parameters.
+   * AdaBelief is sensitive to its parameters and hence a good hyperparameter
+   * selection is necessary as its default may not fit every case.
+   *
+   * The maximum number of iterations refers to the maximum number of
+   * points that are processed (i.e., one iteration equals one point; one
+   * iteration does not equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param beta1 The exponential decay rate for the 1st moment estimates.
+   * @param beta2 The exponential decay rate for the 2nd moment estimates.
+   * @param epsilon A small constant for numerical stability.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  AdaBelief(const double stepSize = 0.001,
+            const size_t batchSize = 32,
+            const double beta1 = 0.9,
+            const double beta2 = 0.999,
+            const double epsilon = 1e-12,
+            const size_t maxIterations = 100000,
+            const double tolerance = 1e-5,
+            const bool shuffle = true,
+            const bool resetPolicy = true,
+            const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using AdaBelief. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the exponential decay rate for the 1st moment estimates.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the exponential decay rate for the 1st moment estimates.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the exponential decay rate for the 2nd moment estimates.
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
+  //! Get the second moment coefficient.
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
+
+  //! Get the value for numerical stability.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used for numerical stability.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+  private:
+  //! The Stochastic Gradient Descent object with AdaBelief policy.
+  SGD<AdaBeliefUpdate> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "ada_belief_impl.hpp"
+
+#endif

include/ensmallen_bits/ada_belief/ada_belief_impl.hpp

@@ -0,0 +1,44 @@
+/**
+ * @file ada_belief_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of AdaBelief class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP
+#define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "ada_belief.hpp"
+
+namespace ens {
+
+inline AdaBelief::AdaBelief(
+    const double stepSize,
+    const size_t batchSize,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle,
+    const bool resetPolicy,
+    const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AdaBeliefUpdate(epsilon, beta1, beta2),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+ #endif

include/ensmallen_bits/ada_belief/ada_belief_update.hpp

@@ -0,0 +1,166 @@
+/**
+ * @file ada_belief_update.hpp
+ * @author Marcus Edel
+ *
+ * AdaBelief optimizer update policy. The intuition for AdaBelief is to adapt
+ * the stepsize according to the "belief" in the current gradient direction.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_UPDATE_HPP
+#define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * The intuition for AdaBelief is to adapt the stepsize according to the
+ * "belief" in the current gradient direction.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{zhuang2020adabelief,
+ *   title         = {AdaBelief Optimizer: Adapting Stepsizes by the Belief in
+ *                    Observed Gradients},
+ *   author        = {Juntang Zhuang and Tommy Tang and Sekhar Tatikonda and
+ *                    Nicha Dvornek and Yifan Ding and Xenophon Papademetris
+ *                    and James S. Duncan},
+ *   year          = {2020},
+ *   eprint        = {2010.07468},
+ *   archivePrefix = {arXiv},
+ * }
+ * @endcode
+ */
+class AdaBeliefUpdate
+{
+ public:
+  /**
+   * Construct the AdaBelief update policy with the given parameters.
+   *
+   * @param epsilon A small constant for numerical stability.
+   * @param beta1 The exponential decay rate for the 1st moment estimates.
+   * @param beta2 The exponential decay rate for the 2nd moment estimates.
+   */
+  AdaBeliefUpdate(const double epsilon = 1e-8,
+                  const double beta1 = 0.9,
+                  const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the value for numerical stability.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used for numerical stability.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the exponential decay rate for the 1st moment estimates.
+  double Beta1() const { return beta1; }
+  //! Modify the exponential decay rate for the 1st moment estimates.
+  double& Beta1() { return beta1; }
+
+  //! Get the exponential decay rate for the 2nd moment estimates.
+  double Beta2() const { return beta2; }
+  //! Modify the exponential decay rate for the 2nd moment estimates.
+  double& Beta2() { return beta2; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    typedef typename MatType::elem_type ElemType;
+
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdaBeliefUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaBeliefUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        epsilon(ElemType(parent.epsilon)),
+        iteration(0)
+    {
+      m.zeros(rows, cols);
+      s.zeros(rows, cols);
+      // Prevent underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
+    }
+
+    /**
+     * Update step for AdaBelief.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++iteration;
+
+      m *= beta1;
+      m += (1 - beta1) * gradient;
+
+      s *= beta2;
+      s += (1 - beta2) * pow(gradient - m, 2) + epsilon;
+
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
+
+      // And update the iterate.
+      iterate -= ((m / biasCorrection1) * ElemType(stepSize)) /
+          (sqrt(s / biasCorrection2) + epsilon);
+    }
+
+   private:
+    //! Instantiated parent object.
+    AdaBeliefUpdate& parent;
+
+    //! The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType s;
+
+    // Parent parameters converted to the element type of the matrix.
+    ElemType beta1;
+    ElemType beta2;
+    ElemType epsilon;
+
+    // The number of iterations.
+    size_t iteration;
+  };
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The xponential decay rate for the 1st moment estimates.
+  double beta1;
+
+  // The exponential decay rate for the 2nd moment estimates.
+  double beta2;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ada_bound/ada_bound.hpp

@@ -107,7 +107,7 @@ class AdaBoundType
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(DecomposableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ada_bound/ada_bound_update.hpp

@@ -56,8 +56,7 @@ class AdaBoundUpdate
     gamma(gamma),
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -87,11 +86,6 @@ class AdaBoundUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -102,6 +96,8 @@ class AdaBoundUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -111,10 +107,24 @@ class AdaBoundUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdaBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0)
+        parent(parent),
+        finalLr(ElemType(parent.finalLr)),
+        gamma(ElemType(parent.gamma)),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        first(true),
+        initialStepSize(0),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
+
+      // Check for underflows in conversions.
+      if (gamma == ElemType(0) && parent.gamma != 0.0)
+        gamma = 10 * std::numeric_limits<ElemType>::epsilon();
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -135,34 +145,30 @@ class AdaBoundUpdate
       if (first)
       {
         first = false;
-        initialStepSize = stepSize;
+        initialStepSize = ElemType(stepSize);
       }
 
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // Decay the first and second moment running average coefficient.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
-      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
-          parent.iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
-          parent.iteration));
+      const ElemType fl = finalLr * ElemType(stepSize) / initialStepSize;
+      const ElemType lower = fl * (1 - 1 / (gamma * iteration + 1));
+      const ElemType upper = fl * (1 + 1 / (gamma * iteration));
 
-       // Applies bounds on actual learning rate.
-      iterate -= arma::clamp((stepSize *
-          std::sqrt(biasCorrection2) / biasCorrection1) / (arma::sqrt(v) +
-          parent.epsilon), lower, upper) % m;
+      // Applies bounds on actual learning rate.
+      iterate -= clamp((ElemType(stepSize) *
+          std::sqrt(biasCorrection2) / biasCorrection1) / (sqrt(v) + epsilon),
+          lower, upper) % m;
     }
 
    private:
@@ -175,11 +181,21 @@ class AdaBoundUpdate
     // The exponential moving average of squared gradient values.
     GradType v;
 
+    // Parameters of the parent, casted to the element type of the problem.
+    ElemType finalLr;
+    ElemType gamma;
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // Whether this is the first call of the Update method.
     bool first;
 
     // The initial (Adam) learning rate.
-    double initialStepSize;
+    ElemType initialStepSize;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -197,9 +213,6 @@ class AdaBoundUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/ada_bound/ams_bound_update.hpp

@@ -56,8 +56,7 @@ class AMSBoundUpdate
       gamma(gamma),
       epsilon(epsilon),
       beta1(beta1),
-      beta2(beta2),
-      iteration(0)
+      beta2(beta2)
   {
     // Nothing to do.
   }
@@ -87,11 +86,6 @@ class AMSBoundUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -102,6 +96,8 @@ class AMSBoundUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -111,11 +107,25 @@ class AMSBoundUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AMSBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0)
+        parent(parent),
+        finalLr(ElemType(parent.finalLr)),
+        gamma(ElemType(parent.gamma)),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        first(true),
+        initialStepSize(0),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
       vImproved.zeros(rows, cols);
+
+      // Check for underflows in conversions.
+      if (gamma == ElemType(0) && parent.gamma != 0.0)
+        gamma = 10 * std::numeric_limits<ElemType>::epsilon();
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -129,44 +139,36 @@ class AMSBoundUpdate
                 const double stepSize,
                 const GradType& gradient)
     {
-      // Convenience typedefs.
-      typedef typename MatType::elem_type ElemType;
-
       // Save the initial step size.
       if (first)
       {
         first = false;
-        initialStepSize = stepSize;
+        initialStepSize = ElemType(stepSize);
       }
 
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // Decay the first and second moment running average coefficient.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
-      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
-          parent.iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
-          parent.iteration));
+      const ElemType fl = finalLr * ElemType(stepSize) / initialStepSize;
+      const ElemType lower = fl * (1 - 1 / (gamma * iteration + 1));
+      const ElemType upper = fl * (1 + 1 / (gamma * iteration));
 
       // Element wise maximum of past and present squared gradients.
-      vImproved = arma::max(vImproved, v);
+      vImproved = max(vImproved, v);
 
       // Applies bounds on actual learning rate.
-      iterate -= arma::clamp((stepSize *
-          std::sqrt(biasCorrection2) / biasCorrection1) /
-          (arma::sqrt(vImproved) + parent.epsilon),  lower, upper) % m;
+      iterate -= clamp((ElemType(stepSize) * std::sqrt(biasCorrection2) /
+          biasCorrection1) / (sqrt(vImproved) + epsilon), lower, upper) % m;
     }
 
    private:
@@ -179,14 +181,24 @@ class AMSBoundUpdate
     // The exponential moving average of squared gradient values.
     GradType v;
 
+    // Parameters of the parent, casted to the element type of the problem.
+    ElemType finalLr;
+    ElemType gamma;
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // Whether this is the first call of the Update method.
     bool first;
 
     // The initial (Adam) learning rate.
-    double initialStepSize;
+    ElemType initialStepSize;
 
     // The optimal squared gradient value.
     GradType vImproved;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -204,9 +216,6 @@ class AMSBoundUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/ada_delta/ada_delta.hpp

@@ -98,7 +98,7 @@ class AdaDelta
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ada_delta/ada_delta_update.hpp

@@ -71,6 +71,8 @@ class AdaDeltaUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD optimizer method before the start
      * of the iteration update process. In AdaDelta update policy, the mean
@@ -82,10 +84,16 @@ class AdaDeltaUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdaDeltaUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        rho(ElemType(parent.rho)),
+        epsilon(ElemType(parent.epsilon))
     {
       meanSquaredGradient.zeros(rows, cols);
       meanSquaredGradientDx.zeros(rows, cols);
+
+      // Check for underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -102,17 +110,17 @@ class AdaDeltaUpdate
                 const GradType& gradient)
     {
       // Accumulate gradient.
-      meanSquaredGradient *= parent.rho;
-      meanSquaredGradient += (1 - parent.rho) * (gradient % gradient);
-      GradType dx = arma::sqrt((meanSquaredGradientDx + parent.epsilon) /
-          (meanSquaredGradient + parent.epsilon)) % gradient;
+      meanSquaredGradient *= rho;
+      meanSquaredGradient += (1 - rho) * (gradient % gradient);
+      GradType dx = sqrt((meanSquaredGradientDx + epsilon) /
+          (meanSquaredGradient + epsilon)) % gradient;
 
       // Accumulate updates.
-      meanSquaredGradientDx *= parent.rho;
-      meanSquaredGradientDx += (1 - parent.rho) * (dx % dx);
+      meanSquaredGradientDx *= rho;
+      meanSquaredGradientDx += (1 - rho) * (dx % dx);
 
       // Apply update.
-      iterate -= (stepSize * dx);
+      iterate -= (ElemType(stepSize) * dx);
     }
 
    private:
@@ -124,6 +132,10 @@ class AdaDeltaUpdate
 
     // The delta mean squared gradient matrix.
     GradType meanSquaredGradientDx;
+
+    // Parameters of the update, converted to the matrix element type.
+    ElemType rho;
+    ElemType epsilon;
   };
 
  private:

include/ensmallen_bits/ada_grad/ada_grad.hpp

@@ -94,7 +94,7 @@ class AdaGrad
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ada_grad/ada_grad_update.hpp

@@ -64,6 +64,8 @@ class AdaGradUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD optimizer before the start of the
      * iteration update process. In AdaGrad update policy, squared gradient
@@ -76,10 +78,14 @@ class AdaGradUpdate
      */
     Policy(AdaGradUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
-        squaredGradient(rows, cols)
+        squaredGradient(rows, cols),
+        epsilon(ElemType(parent.epsilon))
     {
       // Initialize an empty matrix for sum of squares of parameter gradient.
       squaredGradient.zeros();
+      // Detect underflow for epsilon and try to address it.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -96,8 +102,8 @@ class AdaGradUpdate
                 const GradType& gradient)
     {
       squaredGradient += (gradient % gradient);
-      iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) +
-          parent.epsilon);
+      iterate -= (ElemType(stepSize) * gradient) / (sqrt(squaredGradient) +
+          epsilon);
     }
 
    private:
@@ -105,6 +111,8 @@ class AdaGradUpdate
     AdaGradUpdate& parent;
     // The squared gradient matrix.
     GradType squaredGradient;
+    // The epsilon value, converted to the element type of the matrix.
+    ElemType epsilon;
   };
 
  private:

include/ensmallen_bits/ada_sqrt/ada_sqrt.hpp

@@ -0,0 +1,168 @@
+/**
+ * @file ada_sqrt.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the AdaSqrt optimizer. AdaSqrt is an optimizer that
+ * chooses learning rate dynamically by adapting to the data and iteration.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * terms of the 3-clause BSD license. You should have received a copy of the
+ * 3-clause BSD license along with ensmallen. If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_HPP
+
+#include "../sgd/sgd.hpp"
+#include "ada_sqrt_update.hpp"
+
+namespace ens {
+
+/**
+ * AdaSqrt is a modified version of stochastic gradient descent which performs
+ * larger updates for more sparse parameters and smaller updates for less sparse
+ * parameters.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{hu2019secondorder,
+ *   title  = {Second-order Information in First-order Optimization Methods},
+ *   author = {Yuzheng Hu and Licong Lin and Shange Tang},
+ *   year   = {2019},
+ *   eprint = {1912.09926},
+ * }
+ * @endcode
+ *
+ * AdaSqrt can optimize differentiable separable functions.  For more details,
+ * see the documentation on function types included with this distribution or on
+ * the ensmallen website.
+ */
+class AdaSqrt
+{
+ public:
+  /**
+   * Construct the AdaSqrt optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand. The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in one step.
+   * @param epsilon Value used to initialise the squared gradient parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *        function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  AdaSqrt(const double stepSize = 0.01,
+          const size_t batchSize = 32,
+          const double epsilon = 1e-8,
+          const size_t maxIterations = 100000,
+          const double tolerance = 1e-5,
+          const bool shuffle = true,
+          const bool resetPolicy = true,
+          const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using AdaSqrt. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with AdaSqrt policy.
+  SGD<AdaSqrtUpdate> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "ada_sqrt_impl.hpp"
+
+#endif

include/ensmallen_bits/ada_sqrt/ada_sqrt_impl.hpp

@@ -0,0 +1,38 @@
+/**
+ * @file ada_sqrt_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of AdaSqrt optimizer.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_IMPL_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_IMPL_HPP
+
+namespace ens {
+
+inline AdaSqrt::AdaSqrt(const double stepSize,
+                        const size_t batchSize,
+                        const double epsilon,
+                        const size_t maxIterations,
+                        const double tolerance,
+                        const bool shuffle,
+                        const bool resetPolicy,
+                        const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AdaSqrtUpdate(epsilon),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ada_sqrt/ada_sqrt_update.hpp

@@ -0,0 +1,126 @@
+/**
+ * @file ada_sqrt_update.hpp
+ * @author Marcus Edel
+ *
+ * AdaSqrt update for Stochastic Gradient Descent.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_UPDATE_HPP
+#define ENSMALLEN_ADA_SQRT_ADA_SQRT_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * Implementation of the AdaSqrt update policy. AdaSqrt update policy chooses
+ * learning rate dynamically by adapting to the data and iteration.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{hu2019secondorder,
+ *   title  = {Second-order Information in First-order Optimization Methods},
+ *   author = {Yuzheng Hu and Licong Lin and Shange Tang},
+ *   year   = {2019},
+ *   eprint = {1912.09926},
+ * }
+ * @endcode
+ *
+ */
+class AdaSqrtUpdate
+{
+ public:
+  /**
+   * Construct the AdaSqrt update policy with given epsilon parameter.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   */
+  AdaSqrtUpdate(const double epsilon = 1e-8) : epsilon(epsilon)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    typedef typename MatType::elem_type ElemType;
+
+    /**
+     * This constructor is called by the SGD optimizer before the start of the
+     * iteration update process. In AdaSqrt update policy, squared gradient
+     * matrix is initialized to the zeros matrix with the same size as gradient
+     * matrix (see ens::SGD<>).
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaSqrtUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        squaredGradient(rows, cols),
+        epsilon(ElemType(parent.epsilon)),
+        iteration(0)
+    {
+      // Initialize an empty matrix for sum of squares of parameter gradient.
+      squaredGradient.zeros();
+      // Check for underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
+    }
+
+    /**
+     * Update step for SGD. The AdaSqrt update adapts the learning rate by
+     * performing larger updates for more sparse parameters and smaller updates
+     * for less sparse parameters.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      ++iteration;
+
+      squaredGradient += square(gradient);
+
+      iterate -= ElemType(stepSize) * std::sqrt(ElemType(iteration)) *
+          gradient / (squaredGradient + epsilon);
+    }
+
+   private:
+    // Instantiated parent class.
+    AdaSqrtUpdate& parent;
+    // The squared gradient matrix.
+    GradType squaredGradient;
+    // Epsilon converted to the element type of the optimization.
+    ElemType epsilon;
+    // The number of iterations.
+    size_t iteration;
+  };
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/adam/adam.hpp

@@ -120,7 +120,7 @@ class AdamType
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/adam/adam_update.hpp

@@ -52,8 +52,7 @@ class AdamUpdate
              const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -73,11 +72,6 @@ class AdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -88,6 +82,8 @@ class AdamUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -97,10 +93,18 @@ class AdamUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -115,27 +119,26 @@ class AdamUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * square(gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
       /**
        * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
        * following expression is an approximation of the following actual term;
        * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
        */
-      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-          m / (arma::sqrt(v) + parent.epsilon);
+      iterate -= (ElemType(stepSize) *
+          std::sqrt(biasCorrection2) / biasCorrection1) *
+          m / (sqrt(v) + epsilon);
     }
 
    private:
@@ -147,6 +150,14 @@ class AdamUpdate
 
     // The exponential moving average of squared gradient values.
     GradType v;
+
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -158,9 +169,6 @@ class AdamUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/adamax_update.hpp

@@ -30,11 +30,11 @@ namespace ens {
  *
  * @code
  * @article{Kingma2014,
- *   author    = {Diederik P. Kingma and Jimmy Ba},
- *   title     = {Adam: {A} Method for Stochastic Optimization},
- *   journal   = {CoRR},
- *   year      = {2014},
- *   url       = {http://arxiv.org/abs/1412.6980}
+ *   author  = {Diederik P. Kingma and Jimmy Ba},
+ *   title   = {Adam: {A} Method for Stochastic Optimization},
+ *   journal = {CoRR},
+ *   year    = {2014},
+ *   url     = {http://arxiv.org/abs/1412.6980}
  * }
  * @endcode
  */
@@ -54,8 +54,7 @@ class AdaMaxUpdate
                const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -75,11 +74,6 @@ class AdaMaxUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -90,6 +84,8 @@ class AdaMaxUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -99,10 +95,17 @@ class AdaMaxUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdaMaxUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        iteration(0)
     {
       m.zeros(rows, cols);
       u.zeros(rows, cols);
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -117,21 +120,20 @@ class AdaMaxUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
       // Update the exponentially weighted infinity norm.
-      u *= parent.beta2;
-      u = arma::max(u, arma::abs(gradient));
+      u *= beta2;
+      u = max(u, abs(gradient));
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
 
       if (biasCorrection1 != 0)
-        iterate -= (stepSize / biasCorrection1 * m / (u + parent.epsilon));
+        iterate -= (ElemType(stepSize) / biasCorrection1 * m / (u + epsilon));
     }
 
    private:
@@ -141,6 +143,12 @@ class AdaMaxUpdate
     GradType m;
     // The exponentially weighted infinity norm.
     GradType u;
+    // Tuning parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -152,9 +160,6 @@ class AdaMaxUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/amsgrad_update.hpp

@@ -2,7 +2,7 @@
  * @file amsgrad_update.hpp
  * @author Haritha Nair
  *
- * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average
  * optimizer that dynamically adapts over time with guaranteed convergence.
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -25,9 +25,9 @@ namespace ens {
  *
  * @code
  * @article{
- *   title   = {On the convergence of Adam and beyond},
- *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
- *   year    = {2018}
+ *   title = {On the convergence of Adam and beyond},
+ *   url   = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ *   year  = {2018}
  * }
  * @endcode
  */
@@ -47,8 +47,7 @@ class AMSGradUpdate
                 const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -68,11 +67,6 @@ class AMSGradUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -83,6 +77,8 @@ class AMSGradUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -92,11 +88,19 @@ class AMSGradUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AMSGradUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
       vImproved.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -111,25 +115,24 @@ class AMSGradUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
-          parent.iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
       // Element wise maximum of past and present squared gradients.
-      vImproved = arma::max(vImproved, v);
+      vImproved = max(vImproved, v);
 
-      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-                  m / (arma::sqrt(vImproved) + parent.epsilon);
+      iterate -= (ElemType(stepSize) *
+          std::sqrt(biasCorrection2) / biasCorrection1) *
+          m / (sqrt(vImproved) + epsilon);
     }
 
    private:
@@ -144,6 +147,14 @@ class AMSGradUpdate
 
     // The optimal squared gradient value.
     GradType vImproved;
+
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -155,9 +166,6 @@ class AMSGradUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/nadam_update.hpp

@@ -50,8 +50,7 @@ class NadamUpdate
       epsilon(epsilon),
       beta1(beta1),
       beta2(beta2),
-      scheduleDecay(scheduleDecay),
-      iteration(0)
+      scheduleDecay(scheduleDecay)
   {
     // Nothing to do.
   }
@@ -76,11 +75,6 @@ class NadamUpdate
   //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -91,6 +85,8 @@ class NadamUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the optimizer before the start of the
      * iteration update process.
@@ -101,10 +97,18 @@ class NadamUpdate
      */
     Policy(NadamUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
-        cumBeta1(1)
+        cumBeta1(1),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -119,36 +123,34 @@ class NadamUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * gradient % gradient;
+      v *= beta2;
+      v += (1 - beta2) * gradient % gradient;
 
-      double beta1T = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+      ElemType beta1T = beta1 * (1 - ElemType(0.5 *
+          std::pow(0.96, iteration * parent.scheduleDecay)));
 
-      double beta1T1 = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+      ElemType beta1T1 = beta1 * (1 - ElemType(0.5 *
+          std::pow(0.96, (iteration + 1) * parent.scheduleDecay)));
 
       cumBeta1 *= beta1T;
 
-      const double biasCorrection1 = 1.0 - cumBeta1;
-
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
-          parent.iteration);
-
-      const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
+      const ElemType biasCorrection1 = 1 - cumBeta1;
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
+      const ElemType biasCorrection3 = 1 - (cumBeta1 * beta1T1);
 
       /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
        * as arma::sqrt(v) + epsilon
        */
-      iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
-          + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2))
-          / (arma::sqrt(v) + parent.epsilon);
+      iterate -= (ElemType(stepSize) *
+          (((1 - beta1T) / biasCorrection1) * gradient +
+          (beta1T1 / biasCorrection3) * m) * std::sqrt(biasCorrection2)) /
+          (sqrt(v) + epsilon);
     }
 
    private:
@@ -162,7 +164,15 @@ class NadamUpdate
     GradType v;
 
     // The cumulative product of decay coefficients.
-    double cumBeta1;
+    ElemType cumBeta1;
+
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -177,9 +187,6 @@ class NadamUpdate
 
   // The decay parameter for decay coefficients.
   double scheduleDecay;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/nadamax_update.hpp

@@ -50,8 +50,7 @@ class NadaMaxUpdate
       epsilon(epsilon),
       beta1(beta1),
       beta2(beta2),
-      scheduleDecay(scheduleDecay),
-      iteration(0)
+      scheduleDecay(scheduleDecay)
   {
     // Nothing to do.
   }
@@ -76,11 +75,6 @@ class NadaMaxUpdate
   //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -91,6 +85,8 @@ class NadaMaxUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor method is called by the optimizer before the start of
      * the iteration update process.
@@ -101,10 +97,18 @@ class NadaMaxUpdate
      */
     Policy(NadaMaxUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
-        cumBeta1(1)
+        cumBeta1(1),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        iteration(0)
     {
       m.zeros(rows, cols);
       u.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -119,30 +123,30 @@ class NadaMaxUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      u = arma::max(u * parent.beta2, arma::abs(gradient));
+      u = max(u * beta2, abs(gradient));
 
-      double beta1T = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+      ElemType beta1T = beta1 * (1 - ElemType(0.5 *
+          std::pow(0.96, iteration * parent.scheduleDecay)));
 
-      double beta1T1 = parent.beta1 * (1 - (0.5 *
-          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+      ElemType beta1T1 = beta1 * (1 - ElemType(0.5 *
+          std::pow(0.96, (iteration + 1) * parent.scheduleDecay)));
 
       cumBeta1 *= beta1T;
 
-      const double biasCorrection1 = 1.0 - cumBeta1;
-
-      const double biasCorrection2 = 1.0 - (cumBeta1 * beta1T1);
+      const ElemType biasCorrection1 = 1 - cumBeta1;
+      const ElemType biasCorrection2 = 1 - (cumBeta1 * beta1T1);
 
       if ((biasCorrection1 != 0) && (biasCorrection2 != 0))
       {
-         iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
-             + (beta1T1 / biasCorrection2) * m)) / (u + parent.epsilon);
+        iterate -= (ElemType(stepSize) *
+            (((1 - beta1T) / biasCorrection1) * gradient +
+            (beta1T1 / biasCorrection2) * m)) / (u + epsilon);
       }
     }
 
@@ -157,7 +161,15 @@ class NadaMaxUpdate
     GradType u;
 
     // The cumulative product of decay coefficients.
-    double cumBeta1;
+    ElemType cumBeta1;
+
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -172,9 +184,6 @@ class NadaMaxUpdate
 
   // The decay parameter for decay coefficients.
   double scheduleDecay;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/optimisticadam_update.hpp

@@ -27,11 +27,11 @@ namespace ens {
  *
  * @code
  * @article{
- *   author  = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis,
- *              Haoyang Zeng},
- *   title   = {Training GANs with Optimism},
- *   year    = {2017},
- *   url     = {https://arxiv.org/abs/1711.00141}
+ *   author = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis,
+ *             Haoyang Zeng},
+ *   title  = {Training GANs with Optimism},
+ *   year   = {2017},
+ *   url    = {https://arxiv.org/abs/1711.00141}
  * }
  * @endcode
  */
@@ -51,8 +51,7 @@ class OptimisticAdamUpdate
                        const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
-    beta2(beta2),
-    iteration(0)
+    beta2(beta2)
   {
     // Nothing to do.
   }
@@ -72,11 +71,6 @@ class OptimisticAdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the current iteration number.
-  size_t Iteration() const { return iteration; }
-  //! Modify the current iteration number.
-  size_t& Iteration() { return iteration; }
-
   /**
    * The UpdatePolicyType policy classes must contain an internal 'Policy'
    * template class with two template arguments: MatType and GradType.  This is
@@ -87,6 +81,8 @@ class OptimisticAdamUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -96,11 +92,19 @@ class OptimisticAdamUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(OptimisticAdamUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
       g.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -115,24 +119,21 @@ class OptimisticAdamUpdate
                 const GradType& gradient)
     {
       // Increment the iteration counter variable.
-      ++parent.iteration;
+      ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * arma::square(gradient);
+      v *= beta2;
+      v += (1 - beta2) * square(gradient);
 
-      GradType mCorrected = m / (1.0 - std::pow(parent.beta1,
-          parent.iteration));
-      GradType vCorrected = v / (1.0 - std::pow(parent.beta2,
-          parent.iteration));
+      GradType mCorrected = m / (1 - std::pow(beta1, ElemType(iteration)));
+      GradType vCorrected = v / (1 - std::pow(beta2, ElemType(iteration)));
 
-      GradType update = mCorrected /
-          (arma::sqrt(vCorrected) + parent.epsilon);
+      GradType update = mCorrected / (sqrt(vCorrected) + epsilon);
 
-      iterate -= (2 * stepSize * update - stepSize * g);
+      iterate -= (2 * ElemType(stepSize) * update - ElemType(stepSize) * g);
 
       g = std::move(update);
     }
@@ -149,6 +150,14 @@ class OptimisticAdamUpdate
 
     // The previous update.
     GradType g;
+
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
+    // The number of iterations.
+    size_t iteration;
   };
 
  private:
@@ -160,9 +169,6 @@ class OptimisticAdamUpdate
 
   // The second moment coefficient.
   double beta2;
-
-  // The number of iterations.
-  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/agemoea/agemoea.hpp

@@ -0,0 +1,487 @@
+/**
+ * @file agemoea.hpp
+ * @author Satyam Shukla
+ *
+ * AGE-MOEA is a multi-objective optimization algorithm, widely used in
+ * many real-world applications. AGE-MOEA generates offsprings using
+ * crossover and mutation and then selects the next generation according
+ * to non-dominated-sorting and survival score comparison.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_AGEMOEA_AGEMOEA_HPP
+#define ENSMALLEN_AGEMOEA_AGEMOEA_HPP
+
+namespace ens {
+
+/**
+ * This class implements the AGEMOEA algorithm.
+ *
+ * The algorithm works by generating a candidate population from a fixed
+ * starting point. At each stage of optimization, a new population of children
+ * is generated. This new population along with its predecessor is sorted using
+ * non-domination as the metric. Following this, the population is further
+ * segregated in fronts. A new population is generated from these fronts having
+ * size equal to that of the starting population.
+ *
+ * During evolution, two parents are randomly chosen using binary tournament
+ * selection. A pair of children are generated by crossing over these two
+ * candidates followed by mutation.
+ *
+ * The best front (Pareto optimal) is returned by the Optimize() method.
+ *
+ * For more information, see the following:
+ *
+ * @code
+ * @inproceedings{panichella2019adaptive,
+ * title={An adaptive evolutionary algorithm based on non-euclidean geometry for many-objective optimization},
+ * author={Panichella, Annibale},
+ * booktitle={Proceedings of the genetic and evolutionary computation conference},
+ * pages={595--603},
+ * year={2019}
+ * }
+ * @endcode
+ *
+ */
+class AGEMOEA
+{
+ public:
+  /**
+   * Constructor for the AGE-MOEA optimizer.
+   *
+   * The default values provided over here are not necessarily suitable for a
+   * given function. Therefore it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of candidates in the population.
+   *     This should be atleast 4 in size and a multiple of 4.
+   * @param maxGenerations The maximum number of generations allowed for NSGA-II.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param epsilon The minimum difference required to distinguish between
+   *     candidate solutions.
+   * @param eta The distance parameters of the crossover distribution.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  AGEMOEA(const size_t populationSize = 100,
+          const size_t maxGenerations = 2000,
+          const double crossoverProb = 0.6,
+          const double distributionIndex = 20,
+          const double epsilon = 1e-6,
+          const double eta = 20,
+          const arma::vec& lowerBound = arma::zeros(1, 1),
+          const arma::vec& upperBound = arma::ones(1, 1));
+
+  /**
+   * Constructor for the AGE-MOEA optimizer. This constructor provides an overload
+   * to use `lowerBound` and `upperBound` of type double.
+   *
+   * The default values provided over here are not necessarily suitable for a
+   * given function. Therefore it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of candidates in the population.
+   *     This should be atleast 4 in size and a multiple of 4.
+   * @param maxGenerations The maximum number of generations allowed for NSGA-II.
+   * @param crossoverProb The probability that a crossover will occur.
+   * @param distributionIndex The crowding degree of the mutation.
+   * @param epsilon The minimum difference required to distinguish between
+   *     candidate solutions.
+   * @param eta The distance parameters of the crossover distribution
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  AGEMOEA(const size_t populationSize = 100,
+          const size_t maxGenerations = 2000,
+          const double crossoverProb = 0.6,
+          const double distributionIndex = 20,
+          const double epsilon = 1e-6,
+          const double eta = 20,
+          const double lowerBound = 0,
+          const double upperBound = 1);
+
+  /**
+   * Optimize a set of objectives. The initial population is generated using the
+   * starting point. The output is the best generated front.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple objectives.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param objectives Vector of objective functions to optimize for.
+   * @param iterate Starting point.
+   * @param callbacks Callback functions.
+   * @return MatType::elem_type The minimum of the accumulated sum over the
+   *     objective values in the best front.
+   */
+  template<typename MatType,
+           typename... ArbitraryFunctionType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(
+      std::tuple<ArbitraryFunctionType...>& objectives,
+      MatType& iterate,
+      CallbackTypes&&... callbacks);
+
+  /**
+   * Optimize a set of objectives. The initial population is generated using the
+   * starting point. The output is the best generated front.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple objectives.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CubeType The type of cube used to store the front and Pareto set.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param objectives Vector of objective functions to optimize for.
+   * @param iterate Starting point.
+   * @param front The generated front.
+   * @param paretoSet The generated Pareto set.
+   * @param callbacks Callback functions.
+   * @return MatType::elem_type The minimum of the accumulated sum over the
+   *     objective values in the best front.
+   */
+  template<typename MatType,
+           typename CubeType,
+           typename... ArbitraryFunctionType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(
+      std::tuple<ArbitraryFunctionType...>& objectives,
+      MatType& iterate,
+      CubeType& front,
+      CubeType& paretoSet,
+      CallbackTypes&&... callbacks);
+
+  //! Get the population size.
+  size_t PopulationSize() const { return populationSize; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return populationSize; }
+
+  //! Get the maximum number of generations.
+  size_t MaxGenerations() const { return maxGenerations; }
+  //! Modify the maximum number of generations.
+  size_t& MaxGenerations() { return maxGenerations; }
+
+  //! Get the crossover rate.
+  double CrossoverRate() const { return crossoverProb; }
+  //! Modify the crossover rate.
+  double& CrossoverRate() { return crossoverProb; }
+
+  //! Retrieve value of the distribution index.
+  double DistributionIndex() const { return distributionIndex; }
+  //! Modify the value of the distribution index.
+  double& DistributionIndex() { return distributionIndex; }
+
+  //! Retrieve value of eta.
+  double Eta() const { return eta; }
+  //! Modify the value of eta.
+  double& Eta() { return eta; }
+
+  //! Get the tolerance.
+  double Epsilon() const { return epsilon; }
+  //! Modify the tolerance.
+  double& Epsilon() { return epsilon; }
+
+  //! Retrieve value of lowerBound.
+  const arma::vec& LowerBound() const { return lowerBound; }
+  //! Modify value of lowerBound.
+  arma::vec& LowerBound() { return lowerBound; }
+
+  //! Retrieve value of upperBound.
+  const arma::vec& UpperBound() const { return upperBound; }
+  //! Modify value of upperBound.
+  arma::vec& UpperBound() { return upperBound; }
+
+ private:
+  /**
+   * Evaluate objectives for the elite population.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple function types.
+   * @tparam MatType Type of matrix to optimize.
+   * @param population The elite population.
+   * @param objectives The set of objectives.
+   * @param calculatedObjectives Vector to store calculated objectives.
+   */
+  template<std::size_t I = 0,
+           typename InputMatType,
+           typename ObjectiveMatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(std::vector<InputMatType>&,
+                     std::tuple<ArbitraryFunctionType...>&,
+                     std::vector<ObjectiveMatType>&);
+
+  template<std::size_t I = 0,
+           typename InputMatType,
+           typename ObjectiveMatType,
+           typename ...ArbitraryFunctionType>
+  typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+  EvaluateObjectives(std::vector<InputMatType>& population,
+                     std::tuple<ArbitraryFunctionType...>& objectives,
+                     std::vector<ObjectiveMatType>& calculatedObjectives);
+
+  /**
+   * Reproduce candidates from the elite population to generate a new
+   * population.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param objectives The set of objectives.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  template<typename MatType>
+  void BinaryTournamentSelection(std::vector<MatType>& population,
+                                 const MatType& lowerBound,
+                                 const MatType& upperBound);
+
+  /**
+   * Crossover two parents to create a pair of new children.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param childA A newly generated candidate.
+   * @param childB Another newly generated candidate.
+   * @param parentA First parent from elite population.
+   * @param parentB Second parent from elite population.
+   * @param lowerBound The lower bound of the objectives.
+   * @param upperBound The upper bound of the objectives.
+   */
+  template<typename MatType>
+  void Crossover(MatType& childA,
+                 MatType& childB,
+                 const MatType& parentA,
+                 const MatType& parentB,
+                 const MatType& lowerBound,
+                 const MatType& upperBound);
+
+  /**
+   * Mutate the coordinates for a candidate.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param candidate The candidate whose coordinates are being modified.
+   * @param mutationRate The probablity of a mutation to occur.
+   * @param lowerBound Lower bound of the coordinates of the initial population.
+   * @param upperBound Upper bound of the coordinates of the initial population.
+   */
+  template<typename MatType>
+  void Mutate(MatType& candidate,
+              double mutationRate,
+              const MatType& lowerBound,
+              const MatType& upperBound);
+
+  /**
+   * Sort the candidate population using their domination count and the set of
+   * dominated nodes.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param fronts The population is sorted into these Pareto fronts. The first
+   *     front is the best, the second worse and so on.
+   * @param ranks The assigned ranks, used for crowding distance based sorting.
+   * @param calculatedObjectives The previously calculated objectives.
+   */
+  template<typename MatType>
+  void FastNonDominatedSort(
+      std::vector<std::vector<size_t> >& fronts,
+      std::vector<size_t>& ranks,
+      std::vector<arma::Col<typename MatType::elem_type> >&
+          calculatedObjectives);
+
+  /**
+   * Operator to check if one candidate Pareto-dominates the other.
+   *
+   * A candidate is said to dominate the other if it is at least as good as the
+   * other candidate for all the objectives and there exists at least one
+   * objective for which it is strictly better than the other candidate.
+   *
+   * @tparam MatType Type of matrix to optimize.
+   * @param calculatedObjectives The previously calculated objectives.
+   * @param candidateP The candidate being compared from the elite population.
+   * @param candidateQ The candidate being compared against.
+   * @return true if candidateP Pareto dominates candidateQ, otherwise, false.
+   */
+  template<typename MatType>
+  bool Dominates(
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      size_t candidateP,
+      size_t candidateQ);
+
+  /**
+   * Assigns Survival Score metric for sorting.
+   *
+   * @param front The previously generated Pareto fronts.
+   * @param idealPoint The ideal point of teh first front.
+   * @param calculatedObjectives The previously calculated objectives.
+   * @param survivalScore The Survival Score vector to be updated for each
+   *     individual in the population.
+   * @param normalize The normlization vector of the fronts.
+   * @param dimension The dimension of the first front.
+   * @param fNum teh current front index.
+   */
+  template <typename MatType>
+  void SurvivalScoreAssignment(
+      const std::vector<size_t>& front,
+      const arma::Col<typename MatType::elem_type>& idealPoint,
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      std::vector<typename MatType::elem_type>& survivalScore,
+      arma::Col<typename MatType::elem_type>& normalize,
+      typename MatType::elem_type& dimension,
+      size_t fNum);
+
+  /**
+   * The operator used in the AGE-MOEA survival score based sorting.
+   *
+   * If a candidate has a lower rank then it is preferred.
+   * Otherwise, if the ranks are equal then the candidate with the larger
+   * Survival Score is preferred.
+   *
+   * @param idxP The index of the first cadidate from the elite population being
+   *     sorted.
+   * @param idxQ The index of the second cadidate from the elite population
+   *     being sorted.
+   * @param ranks The previously calculated ranks.
+   * @param survivalScore The Survival score for each individual in
+   *     the population.
+   * @return true if the first candidate is preferred, otherwise, false.
+   */
+  template<typename MatType>
+  bool SurvivalScoreOperator(
+      size_t idxP,
+      size_t idxQ,
+      const std::vector<size_t>& ranks,
+      const std::vector<typename MatType::elem_type>& survivalScore);
+
+  /**
+   * Normalizes the front given the extreme points in the current front.
+   *
+   * @tparam The type of population datapoints.
+   * @param calculatedObjectives The current population evaluated objectives.
+   * @param normalization The normalizing vector.
+   * @param front The previously generated Pareto front.
+   * @param extreme The indexes of the extreme points in the front.
+   */
+  template <typename MatType>
+  void NormalizeFront(
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      arma::Col<typename MatType::elem_type>& normalization,
+      const std::vector<size_t>& front,
+      const arma::Row<size_t>& extreme);
+
+  /**
+   * Get the geometry information p of Lp norm (p > 0).
+   *
+   * @param calculatedObjectives The current population evaluated objectives.
+   * @param front The previously generated Pareto fronts.
+   * @param extreme The indexes of the extreme points in the front.
+   * @return The variable p in the Lp norm that best fits the geometry of the
+   *     current front.
+   */
+  template <typename MatType>
+  typename MatType::elem_type GetGeometry(
+      std::vector<arma::Col<typename MatType::elem_type> >&
+          calculatedObjectives,
+      const std::vector<size_t>& front,
+      const arma::Row<size_t>& extreme);
+
+  /**
+   * Finds the pairwise Lp distance between all the points in the front.
+   *
+   * @param final The current population evaluated objectives.
+   * @param calculatedObjectives The current population evaluated objectives.
+   * @param front The front of the current generation.
+   * @param dimension The calculated dimension of the front.
+   */
+  template <typename MatType>
+  void PairwiseDistance(
+      MatType& final,
+      std::vector<arma::Col<typename MatType::elem_type> >&
+          calculatedObjectives,
+      const std::vector<size_t>& front,
+      const typename MatType::elem_type dimension);
+
+  /**
+   * Finding the indexes of the extreme points in the front.
+   *
+   * @param indexes vector containing the slected indexes.
+   * @param calculatedObjectives The current population objectives.
+   * @param front The front of the current generation.
+   */
+  template <typename MatType>
+  void FindExtremePoints(
+      arma::Row<size_t>& indexes,
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      const std::vector<size_t>& front);
+
+  /**
+   * Finding the distance of each point in the front from the line formed
+   * by pointA and pointB.
+   *
+   * @param distance The vector containing the distances of the points in the
+   *    from from the line.
+   * @param calculatedObjectives Reference to the current population evaluated
+   *    objectives.
+   * @param front The front of the current generation(indices of population).
+   * @param pointA The first point on the line.
+   * @param pointB The second point on the line.
+   */
+  template <typename MatType>
+  void PointToLineDistance(
+      arma::Row<typename MatType::elem_type>& distances,
+      std::vector<arma::Col<typename MatType::elem_type> >&
+          calculatedObjectives,
+      const std::vector<size_t>& front,
+      const arma::Col<typename MatType::elem_type>& pointA,
+      const arma::Col<typename MatType::elem_type>& pointB);
+
+  /**
+   * Find the Diversity score corresponding the solution S using the selected
+   * set.
+   *
+   * @param selected The current selected set.
+   * @param pairwiseDistance The current pairwise distance for the whole front.
+   * @param S The relative index of S being considered within the front.
+   * @return The diversity score for S which the sum of the two smallest
+   *     elements.
+  */
+ template <typename MatType>
+ typename MatType::elem_type DiversityScore(std::set<size_t>& selected,
+                                            const MatType& pairwiseDistance,
+                                            size_t S);
+
+  //! The number of objectives being optimised for.
+  size_t numObjectives;
+
+  //! The numbeer of variables used per objectives.
+  size_t numVariables;
+
+  //! The number of candidates in the population.
+  size_t populationSize;
+
+  //! Maximum number of generations before termination criteria is met.
+  size_t maxGenerations;
+
+  //! Probability that crossover will occur.
+  double crossoverProb;
+
+  //! The crowding degree of the mutation. Higher value produces a mutant
+  //! resembling its parent.
+  double distributionIndex;
+
+  //! The tolerance for termination.
+  double epsilon;
+
+  //! The distance parameters of the crossover distribution.
+  double eta;
+
+  //! Lower bound of the initial swarm.
+  arma::vec lowerBound;
+
+  //! Upper bound of the initial swarm.
+  arma::vec upperBound;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "agemoea_impl.hpp"
+
+#endif

include/ensmallen_bits/agemoea/agemoea_impl.hpp

@@ -0,0 +1,848 @@
+/**
+ * @file agemoea_impl.hpp
+ * @author Satyam Shukla
+ *
+ * Implementation of the AGEMOEA algorithm. Used for multi-objective
+ * optimization problems on arbitrary functions.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more Information.
+ */
+
+#ifndef ENSMALLEN_AGEMOEA_AGEMOEA_IMPL_HPP
+#define ENSMALLEN_AGEMOEA_AGEMOEA_IMPL_HPP
+
+#include "agemoea.hpp"
+#include <assert.h>
+
+namespace ens {
+
+inline AGEMOEA::AGEMOEA(const size_t populationSize,
+                        const size_t maxGenerations,
+                        const double crossoverProb,
+                        const double distributionIndex,
+                        const double epsilon,
+                        const double eta,
+                        const arma::vec& lowerBound,
+                        const arma::vec& upperBound) :
+    numObjectives(0),
+    numVariables(0),
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    distributionIndex(distributionIndex),
+    epsilon(epsilon),
+    eta(eta),
+    lowerBound(lowerBound),
+    upperBound(upperBound)
+{ /* Nothing to do here. */ }
+
+inline AGEMOEA::AGEMOEA(const size_t populationSize,
+                        const size_t maxGenerations,
+                        const double crossoverProb,
+                        const double distributionIndex,
+                        const double epsilon,
+                        const double eta,
+                        const double lowerBound,
+                        const double upperBound) :
+    numObjectives(0),
+    numVariables(0),
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverProb(crossoverProb),
+    distributionIndex(distributionIndex),
+    epsilon(epsilon),
+    eta(eta),
+    lowerBound(lowerBound * arma::ones(1, 1)),
+    upperBound(upperBound * arma::ones(1, 1))
+{ /* Nothing to do here. */ }
+
+//! Optimize the function.
+template<typename MatType,
+         typename... ArbitraryFunctionType,
+         typename... CallbackTypes>
+typename MatType::elem_type AGEMOEA::Optimize(
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
+{
+  typedef typename ForwardType<MatType>::bcube CubeType;
+  CubeType paretoFront, paretoSet;
+  return Optimize(objectives, iterateIn, paretoFront, paretoSet,
+      std::forward<CallbackTypes>(callbacks)...);
+}
+
+//! Optimize the function.
+template<typename MatType,
+         typename CubeType,
+         typename... ArbitraryFunctionType,
+         typename... CallbackTypes>
+typename MatType::elem_type AGEMOEA::Optimize(
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    MatType& iterateIn,
+    CubeType& paretoFrontIn,
+    CubeType& paretoSetIn,
+    CallbackTypes&&... callbacks)
+{
+  // Make sure for evolution to work at least four candidates are present.
+  if (populationSize < 4 && populationSize % 4 != 0)
+  {
+    throw std::logic_error("AGEMOEA::Optimize(): population size should be at"
+        " least 4, and, a multiple of 4!");
+  }
+
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename ForwardType<MatType>::bcol BaseColType;
+  typedef typename ForwardType<CubeType>::bmat CubeBaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType...,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  // Check if lower bound is a vector of a single dimension.
+  if (lowerBound.n_rows == 1)
+    lowerBound = lowerBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check if upper bound is a vector of a single dimension.
+  if (upperBound.n_rows == 1)
+    upperBound = upperBound(0, 0) * arma::ones(iterate.n_rows, iterate.n_cols);
+
+  // Check the dimensions of lowerBound and upperBound.
+  assert(lowerBound.n_rows == iterate.n_rows && "The dimensions of "
+      "lowerBound are not the same as the dimensions of iterate.");
+  assert(upperBound.n_rows == iterate.n_rows && "The dimensions of "
+      "upperBound are not the same as the dimensions of iterate.");
+
+  numObjectives = sizeof...(ArbitraryFunctionType);
+  numVariables = iterate.n_rows;
+
+  // Cache calculated objectives.
+  std::vector<BaseColType> calculatedObjectives(populationSize);
+
+  // Population size reserved to 2 * populationSize + 1 to accommodate
+  // for the size of intermediate candidate population.
+  std::vector<BaseMatType> population;
+  population.reserve(2 * populationSize + 1);
+
+  // Pareto fronts, initialized during non-dominated sorting.
+  // Stores indices of population belonging to a certain front.
+  std::vector<std::vector<size_t> > fronts;
+  // Initialised in SurvivalScoreAssignment.
+  std::vector<ElemType> survivalScore;
+  // Initialised during non-dominated sorting.
+  std::vector<size_t> ranks;
+
+  //! Useful temporaries for float-like comparisons.
+  const BaseMatType castedLowerBound = conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = conv_to<BaseMatType>::from(upperBound);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // Generate the population based on a uniform distribution around the given
+  // starting point.
+  for (size_t i = 0; i < populationSize; i++)
+  {
+    population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
+        iterate.n_cols) - ElemType(0.5) + iterate);
+
+    // Constrain all genes to be within bounds.
+    population[i] = min(max(population[i], castedLowerBound),
+        castedUpperBound);
+  }
+
+  Info << "AGEMOEA initialized successfully. Optimization started." << std::endl;
+
+  // Iterate until maximum number of generations is obtained.
+  Callback::BeginOptimization(*this, objectives, iterate, callbacks...);
+
+  for (size_t generation = 1; generation <= maxGenerations && !terminate; generation++)
+  {
+    // Create new population of candidate from the present elite population.
+    // Have P_t, generate G_t using P_t.
+    BinaryTournamentSelection(population, castedLowerBound, castedUpperBound);
+
+    // Evaluate the objectives for the new population.
+    calculatedObjectives.resize(population.size());
+    std::fill(calculatedObjectives.begin(), calculatedObjectives.end(),
+        BaseColType(numObjectives, GetFillType<MatType>::zeros));
+    EvaluateObjectives(population, objectives, calculatedObjectives);
+
+    // Perform fast non dominated sort on P_t ∪ G_t.
+    ranks.resize(population.size());
+    FastNonDominatedSort<BaseMatType>(fronts, ranks, calculatedObjectives);
+
+    arma::Col<ElemType> idealPoint(calculatedObjectives[fronts[0][0]]);
+    for (size_t index = 1; index < fronts[0].size(); index++)
+    {
+      idealPoint = min(idealPoint, calculatedObjectives[fronts[0][index]]);
+    }
+
+    // Perform survival score assignment.
+    survivalScore.resize(population.size());
+    std::fill(survivalScore.begin(), survivalScore.end(), 0.);
+    ElemType dimension;
+    BaseColType normalize(numObjectives, GetFillType<MatType>::zeros);
+    for (size_t fNum = 0; fNum < fronts.size(); fNum++)
+    {
+      SurvivalScoreAssignment<BaseMatType>(fronts[fNum], idealPoint,
+      calculatedObjectives, survivalScore, normalize, dimension, fNum);
+    }
+
+    // Sort based on survival score.
+    std::sort(population.begin(), population.end(),
+      [this, ranks, survivalScore, population]
+        (BaseMatType candidateP, BaseMatType candidateQ)
+          {
+            size_t idxP{}, idxQ{};
+            for (size_t i = 0; i < population.size(); i++)
+            {
+              if (approx_equal(population[i], candidateP, "absdiff",
+                  ElemType(epsilon)))
+                idxP = i;
+
+              if (approx_equal(population[i], candidateQ, "absdiff",
+                  ElemType(epsilon)))
+                idxQ = i;
+            }
+
+            return SurvivalScoreOperator<BaseMatType>(idxP, idxQ, ranks,
+                survivalScore);
+          }
+    );
+
+    // Yield a new population P_{t+1} of size populationSize.
+    // Discards unfit population from the R_{t} to yield P_{t+1}.
+    population.resize(populationSize);
+
+    terminate |= Callback::GenerationalStepTaken(*this, objectives, iterate,
+        calculatedObjectives, fronts, callbacks...);
+  }
+  EvaluateObjectives(population, objectives, calculatedObjectives);
+  // Set the candidates from the Pareto Set as the output.
+  paretoSetIn.set_size(population[0].n_rows, population[0].n_cols,
+      population.size());
+  // The Pareto Set is stored, can be obtained via ParetoSet() getter.
+  for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
+  {
+    paretoSetIn.slice(solutionIdx) =
+      conv_to<CubeBaseMatType>::from(population[solutionIdx]);
+  }
+
+  // Set the candidates from the Pareto Front as the output.
+  paretoFrontIn.set_size(calculatedObjectives[0].n_rows,
+      calculatedObjectives[0].n_cols, population.size());
+  for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
+  {
+    paretoFrontIn.slice(solutionIdx) =
+      conv_to<CubeBaseMatType>::from(calculatedObjectives[solutionIdx]);
+  }
+
+  // Assign iterate to first element of the Pareto Set.
+  iterate = population[fronts[0][0]];
+
+  Callback::EndOptimization(*this, objectives, iterate, callbacks...);
+
+  ElemType performance = std::numeric_limits<ElemType>::max();
+
+  for (const BaseColType& objective: calculatedObjectives)
+    if (accu(objective) < performance)
+      performance = accu(objective);
+
+  return performance;
+}
+
+//! No objectives to evaluate.
+template<std::size_t I,
+         typename InputMatType,
+         typename ObjectiveMatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
+AGEMOEA::EvaluateObjectives(
+    std::vector<InputMatType>&,
+    std::tuple<ArbitraryFunctionType...>&,
+    std::vector<ObjectiveMatType>&)
+{
+  // Nothing to do here.
+}
+
+//! Evaluate the objectives for the entire population.
+template<std::size_t I,
+         typename InputMatType,
+         typename ObjectiveMatType,
+         typename ...ArbitraryFunctionType>
+typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
+AGEMOEA::EvaluateObjectives(
+    std::vector<InputMatType>& population,
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    std::vector<ObjectiveMatType>& calculatedObjectives)
+{
+  for (size_t i = 0; i < population.size(); i++)
+  {
+    calculatedObjectives[i](I) = std::get<I>(objectives).Evaluate(population[i]);
+    EvaluateObjectives<I+1, InputMatType, ObjectiveMatType,
+        ArbitraryFunctionType...>(population, objectives,
+        calculatedObjectives);
+  }
+}
+
+//! Reproduce and generate new candidates.
+template<typename InputMatType>
+inline void AGEMOEA::BinaryTournamentSelection(std::vector<InputMatType>& population,
+                                               const InputMatType& lowerBound,
+                                               const InputMatType& upperBound)
+{
+  std::vector<InputMatType> children;
+
+  while (children.size() < population.size())
+  {
+    // Choose two random parents for reproduction from the elite population.
+    size_t indexA = arma::randi<size_t>(
+        arma::distr_param(0, populationSize - 1));
+    size_t indexB = arma::randi<size_t>(
+        arma::distr_param(0, populationSize - 1));
+
+    // Make sure that the parents differ.
+    if (indexA == indexB)
+    {
+      if (indexB < populationSize - 1)
+        indexB++;
+      else
+        indexB--;
+    }
+
+    // Initialize the children to the respective parents.
+    InputMatType childA = population[indexA], childB = population[indexB];
+
+    if (arma::randu() <= crossoverProb)
+      Crossover(childA, childB, population[indexA], population[indexB],
+                lowerBound, upperBound);
+
+    Mutate(childA, 1.0 / static_cast<double>(numVariables),
+        lowerBound, upperBound);
+    Mutate(childB, 1.0 / static_cast<double>(numVariables),
+        lowerBound, upperBound);
+
+    // Add the children to the candidate population.
+    children.push_back(childA);
+    children.push_back(childB);
+  }
+
+  // Add the candidates to the elite population.
+  population.insert(std::end(population), std::begin(children),
+      std::end(children));
+}
+
+//! Perform simulated binary crossover (SBX) of genes for the children.
+template<typename InputMatType>
+inline void AGEMOEA::Crossover(InputMatType& childA,
+                               InputMatType& childB,
+                               const InputMatType& parentA,
+                               const InputMatType& parentB,
+                               const InputMatType& lowerBound,
+                               const InputMatType& upperBound)
+{
+  typedef typename InputMatType::elem_type ElemType;
+  typedef typename ForwardType<InputMatType>::bcube BaseCubeType;
+  typedef typename ForwardType<InputMatType>::umat UMatType;
+
+  // Generates a child from two parent individuals
+  // according to the polynomial probability distribution.
+  BaseCubeType parents(parentA.n_rows,
+    parentA.n_cols, 2);
+  parents.slice(0) = parentA;
+  parents.slice(1) = parentB;
+  InputMatType current_min = min(parents, 2);
+  InputMatType current_max = max(parents, 2);
+
+  if (accu(parentA - parentB < ElemType(1e-14)))
+  {
+    childA = parentA;
+    childB = parentB;
+    return;
+  }
+  InputMatType current_diff = current_max - current_min;
+  current_diff.transform( [](ElemType val)
+      { return (val < ElemType(1e-10) ? ElemType(1e-10) : val); } );
+
+  // Calculating beta used for the final crossover.
+  InputMatType beta1 = 1 + 2 * (current_min - lowerBound) / current_diff;
+  InputMatType beta2 = 1 + 2 * (upperBound - current_max) / current_diff;
+  InputMatType alpha1 = 2 - pow(beta1, -(eta + 1));
+  InputMatType alpha2 = 2 - pow(beta2, -(eta + 1));
+
+  InputMatType us(size(alpha1), GetFillType<InputMatType>::randu);
+
+  UMatType mask1 = us > (1 / alpha1);
+  InputMatType betaq1 = pow(us % alpha1, 1. / (eta + 1));
+  betaq1 = betaq1 % (mask1 != 1) + pow((1 / (2 - us % alpha1)),
+      1 / (eta + 1)) % mask1;
+  UMatType mask2 = us > (1 / alpha2);
+  InputMatType betaq2 = pow(us % alpha2, 1 / (eta + 1));
+  betaq2 = betaq2 % (mask1 != 1) + pow((1 / (2 - us % alpha2)),
+      1 / (eta + 1)) % mask2;
+
+  // Variables after the cross over for all of them.
+  InputMatType c1 = ((current_min + current_max) - betaq1 % current_diff) / 2;
+  InputMatType c2 = ((current_min + current_max) + betaq2 % current_diff) / 2;
+  c1 = min(max(c1, lowerBound), upperBound);
+  c2 = min(max(c2, lowerBound), upperBound);
+
+  // Decision for the crossover between the two parents for each variable.
+  us.randu();
+  childA = parentA % (us <= ElemType(0.5));
+  childB = parentB % (us <= ElemType(0.5));
+  us.randu();
+  childA = childA + c1 % ((us <= ElemType(0.5)) % (childA == 0));
+  childA = childA + c2 % ((us >  ElemType(0.5)) % (childA == 0));
+  childB = childB + c2 % ((us <= ElemType(0.5)) % (childB == 0));
+  childB = childB + c1 % ((us >  ElemType(0.5)) % (childB == 0));
+}
+
+//! Perform Polynomial mutation of the candidate.
+template<typename MatType>
+inline void AGEMOEA::Mutate(MatType& candidate,
+                            double mutationRate,
+                            const MatType& lowerBound,
+                            const MatType& upperBound)
+{
+  const size_t numVariables = candidate.n_rows;
+  for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+  {
+    // Should this gene be mutated?
+    if (arma::randu() > mutationRate)
+      continue;
+
+    const double geneRange = upperBound(geneIdx) - lowerBound(geneIdx);
+    // Normalised distance from the bounds.
+    const double lowerDelta = (candidate(geneIdx)
+        - lowerBound(geneIdx)) / geneRange;
+    const double upperDelta = (upperBound(geneIdx)
+        - candidate(geneIdx)) / geneRange;
+    const double mutationPower = 1. / (distributionIndex + 1.0);
+    const double rand = arma::randu();
+    double value, perturbationFactor;
+    if (rand < 0.5)
+    {
+      value = 2.0 * rand + (1.0 - 2.0 * rand) *
+          std::pow(upperDelta, distributionIndex + 1.0);
+      perturbationFactor = std::pow(value, mutationPower) - 1.0;
+    }
+    else
+    {
+      value = 2.0 * (1.0 - rand) + 2.0 *(rand - 0.5) *
+          std::pow(lowerDelta, distributionIndex + 1.0);
+      perturbationFactor = 1.0 - std::pow(value, mutationPower);
+    }
+
+    candidate(geneIdx) +=
+        typename MatType::elem_type(perturbationFactor * geneRange);
+  }
+  //! Enforce bounds.
+  candidate = min(max(candidate, lowerBound), upperBound);
+}
+
+template <typename MatType>
+inline void AGEMOEA::NormalizeFront(
+      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      arma::Col<typename MatType::elem_type>& normalization,
+      const std::vector<size_t>& front,
+      const arma::Row<size_t>& extreme)
+{
+  arma::Mat<typename MatType::elem_type> vectorizedObjectives(numObjectives,
+      front.size());
+  arma::Mat<typename MatType::elem_type> vectorizedExtremes(numObjectives,
+      extreme.n_elem);
+  for (size_t i = 0; i < front.size(); i++)
+  {
+    vectorizedObjectives.col(i) = calculatedObjectives[front[i]];
+  }
+  for (size_t i = 0; i < extreme.n_elem; i++)
+  {
+    vectorizedExtremes.col(i) = calculatedObjectives[front[extreme[i]]];
+  }
+
+  if (front.size() < numObjectives)
+  {
+    normalization = arma::max(vectorizedObjectives, 1);
+    return;
+  }
+  arma::Col<typename MatType::elem_type> temp;
+  arma::uvec unique = arma::find_unique(extreme);
+  if (extreme.n_elem != unique.n_elem)
+  {
+    normalization = arma::max(vectorizedObjectives, 1);
+    return;
+  }
+  arma::Col<typename MatType::elem_type> one(extreme.n_elem, arma::fill::ones);
+  arma::Col<typename MatType::elem_type> hyperplane(numObjectives, arma::fill::zeros);
+  try{
+    hyperplane = arma::solve(
+    vectorizedExtremes.t(), one);
+  }
+  catch(...)
+  {
+    normalization = arma::max(vectorizedObjectives, 1);
+    normalization = normalization + (normalization == 0);
+    return;
+  }
+  if (hyperplane.has_inf() || hyperplane.has_nan() || (arma::accu(hyperplane < 0.0) > 0))
+  {
+    normalization = arma::max(vectorizedObjectives, 1);
+  }
+  else
+  {
+    normalization = 1. / hyperplane;
+    if (normalization.has_inf() || normalization.has_nan())
+    {
+      normalization = arma::max(vectorizedObjectives, 1);
+    }
+  }
+  normalization = normalization + (normalization == 0);
+}
+
+template <typename MatType>
+inline typename MatType::elem_type AGEMOEA::GetGeometry(
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    const std::vector<size_t>& front,
+    const arma::Row<size_t>& extreme)
+{
+  typedef typename MatType::elem_type ElemType;
+
+  arma::Row<ElemType> d;
+  arma::Col<ElemType> zero(numObjectives, arma::fill::zeros);
+  arma::Col<ElemType> one(numObjectives, arma::fill::ones);
+
+  PointToLineDistance<MatType>(d, calculatedObjectives, front, zero, one);
+
+  for (size_t i = 0; i < extreme.size(); i++)
+  {
+    d[extreme[i]] = arma::Datum<ElemType>::inf;
+  }
+
+  size_t index = arma::index_min(d);
+  ElemType avg = accu(calculatedObjectives[front[index]]) / numObjectives;
+  ElemType p = std::log(ElemType(numObjectives)) / std::log(1 / avg);
+  if (p <= ElemType(0.1) || std::isnan(p))
+    p = 1;
+
+  return p;
+}
+
+//! Pairwise distance for each point in the given front.
+template <typename MatType>
+inline void AGEMOEA::PairwiseDistance(
+    MatType& f,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    const std::vector<size_t>& front,
+    const typename MatType::elem_type dimension)
+{
+  for (size_t i = 0; i < front.size(); i++)
+  {
+    for (size_t j = i + 1; j < front.size(); j++)
+    {
+      f(i, j) = std::pow(accu(pow(abs(
+          calculatedObjectives[front[i]] - calculatedObjectives[front[j]]),
+          dimension)), 1 / dimension);
+      f(j, i) = f(i, j);
+    }
+  }
+}
+
+//! Find the index of the of the extreme points in the given front.
+template <typename MatType>
+void AGEMOEA::FindExtremePoints(
+    arma::Row<size_t>& indexes,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    const std::vector<size_t>& front)
+{
+  typedef typename MatType::elem_type ElemType;
+
+  if (numObjectives >= front.size())
+  {
+    indexes = arma::linspace<arma::Row<size_t>>(0, front.size() - 1, front.size());
+    return;
+  }
+
+  arma::Mat<ElemType> W(numObjectives, numObjectives, arma::fill::eye);
+  W = W + 1e-6;
+  std::vector<bool> selected(front.size());
+  arma::Col<ElemType> z(numObjectives, arma::fill::zeros);
+  arma::Row<ElemType> dists;
+  for (size_t i = 0; i < numObjectives; i++)
+  {
+    PointToLineDistance<MatType>(dists, calculatedObjectives, front, z, W.col(i));
+    for (size_t j = 0; j < front.size(); j++)
+      if (selected[j]){dists[j] = arma::datum::inf;}
+    indexes[i] = dists.index_min();
+    selected[dists.index_min()] = true;
+  }
+}
+
+//! Find the distance of a front from a line formed by two points.
+template <typename MatType>
+void AGEMOEA::PointToLineDistance(
+    arma::Row<typename MatType::elem_type>& distances,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    const std::vector<size_t>& front,
+    const arma::Col<typename MatType::elem_type>& pointA,
+    const arma::Col<typename MatType::elem_type>& pointB)
+{
+  typedef typename MatType::elem_type ElemType;
+  arma::Row<ElemType> distancesTemp(front.size());
+  arma::Col<ElemType> ba = pointB - pointA;
+  arma::Col<ElemType> pa;
+
+  for (size_t i = 0; i < front.size(); i++)
+  {
+    size_t ind = front[i];
+
+    pa = (calculatedObjectives[ind] - pointA);
+    double t = arma::dot(pa, ba) / arma::dot(ba, ba);
+    distancesTemp[i] = arma::accu(arma::pow((pa - t * ba), 2));
+  }
+  distances = distancesTemp;
+}
+
+//! Sort population into Pareto fronts.
+template<typename MatType>
+inline void AGEMOEA::FastNonDominatedSort(
+    std::vector<std::vector<size_t> >& fronts,
+    std::vector<size_t>& ranks,
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+{
+  std::map<size_t, size_t> dominationCount;
+  std::map<size_t, std::set<size_t> > dominated;
+
+  // Reset and initialize fronts.
+  fronts.clear();
+  fronts.push_back(std::vector<size_t>());
+
+  for (size_t p = 0; p < calculatedObjectives.size(); p++)
+  {
+    dominated[p] = std::set<size_t>();
+    dominationCount[p] = 0;
+
+    for (size_t q = 0; q < calculatedObjectives.size(); q++)
+    {
+      if (Dominates<MatType>(calculatedObjectives, p, q))
+        dominated[p].insert(q);
+      else if (Dominates<MatType>(calculatedObjectives, q, p))
+        dominationCount[p] += 1;
+    }
+
+    if (dominationCount[p] == 0)
+    {
+      ranks[p] = 0;
+      fronts[0].push_back(p);
+    }
+  }
+
+  size_t i = 0;
+
+  while (!fronts[i].empty())
+  {
+    std::vector<size_t> nextFront;
+
+    for (size_t p: fronts[i])
+    {
+      for (size_t q: dominated[p])
+      {
+        dominationCount[q]--;
+
+        if (dominationCount[q] == 0)
+        {
+          ranks[q] = i + 1;
+          nextFront.push_back(q);
+        }
+      }
+    }
+
+    i++;
+    fronts.push_back(nextFront);
+  }
+  // Remove the empty final set.
+  fronts.pop_back();
+}
+
+//! Check if a candidate Pareto dominates another candidate.
+template<typename MatType>
+inline bool AGEMOEA::Dominates(
+    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+    size_t candidateP,
+    size_t candidateQ)
+{
+  bool allBetterOrEqual = true;
+  bool atleastOneBetter = false;
+  size_t n_objectives = calculatedObjectives[0].n_elem;
+
+  for (size_t i = 0; i < n_objectives; i++)
+  {
+    // P is worse than Q for the i-th objective function.
+    if (calculatedObjectives[candidateP](i) > calculatedObjectives[candidateQ](i))
+      allBetterOrEqual = false;
+
+    // P is better than Q for the i-th objective function.
+    else if (calculatedObjectives[candidateP](i) <
+        calculatedObjectives[candidateQ](i))
+      atleastOneBetter = true;
+  }
+
+  return allBetterOrEqual && atleastOneBetter;
+}
+
+//! Assign diversity score for a given point and the selected set.
+template <typename MatType>
+inline typename MatType::elem_type AGEMOEA::DiversityScore(
+    std::set<size_t>& selected,
+    const MatType& pairwiseDistance,
+    size_t S)
+{
+  typedef typename MatType::elem_type ElemType;
+  ElemType m = arma::datum::inf;
+  ElemType m1 = arma::datum::inf;
+  std::set<size_t>::iterator it;
+  for (it = selected.begin(); it != selected.end(); it++)
+  {
+    if (*it == S){ continue; }
+    if (pairwiseDistance(S, *it) < m)
+    {
+      m1 = m;
+      m = pairwiseDistance(S, *it);
+    }
+    else if (pairwiseDistance(S, *it) < m1)
+    {
+      m1 = pairwiseDistance(S, *it);
+    }
+  }
+  m1 = (m1 == arma::datum::inf) ? 0 : m1;
+  m = (m == arma::datum::inf) ? 0 : m;
+  return m + m1;
+}
+
+//! Assign survival score for a front of the population.
+template <typename MatType>
+inline void AGEMOEA::SurvivalScoreAssignment(
+    const std::vector<size_t>& front,
+    const arma::Col<typename MatType::elem_type>& idealPoint,
+    std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+    std::vector<typename MatType::elem_type>& survivalScore,
+    arma::Col<typename MatType::elem_type>& normalize,
+    typename MatType::elem_type& dimension,
+    size_t fNum)
+{
+  typedef typename MatType::elem_type ElemType;
+
+  // Calculations for the first front.
+  if (fNum == 0)
+  {
+    if (front.size() < numObjectives)
+    {
+      dimension = 1;
+      arma::Row<size_t> extreme(numObjectives, arma::fill::zeros);
+      NormalizeFront<MatType>(calculatedObjectives, normalize, front, extreme);
+      return;
+    }
+
+    for (size_t index = 0; index < front.size(); index++)
+    {
+      calculatedObjectives[front[index]] = calculatedObjectives[front[index]]
+          - idealPoint;
+    }
+
+    arma::Row<size_t> extreme(numObjectives, arma::fill::zeros);
+    FindExtremePoints<MatType>(extreme, calculatedObjectives, front);
+    NormalizeFront<MatType>(calculatedObjectives, normalize, front, extreme);
+
+    for (size_t index = 0; index < front.size(); index++)
+    {
+      calculatedObjectives[front[index]] = calculatedObjectives[front[index]]
+          / normalize;
+    }
+
+    std::set<size_t> selected;
+    std::set<size_t> remaining;
+
+    // Create the selected and remaining sets.
+    for (size_t index: extreme)
+    {
+      selected.insert(index);
+      survivalScore[front[index]] = arma::Datum<ElemType>::inf;
+    }
+
+    dimension = GetGeometry<MatType>(calculatedObjectives, front, extreme);
+    for (size_t i = 0; i < front.size(); i++)
+    {
+      if (selected.count(i) == 0)
+      {
+        remaining.insert(i);
+      }
+    }
+
+    arma::Mat<ElemType> pairwise(front.size(), front.size(), arma::fill::zeros);
+    PairwiseDistance<MatType>(pairwise, calculatedObjectives, front, dimension);
+    arma::Row<typename MatType::elem_type> value(front.size(),
+        arma::fill::zeros);
+
+    // Calculate the diversity and proximity score.
+    for (size_t i = 0; i < front.size(); i++)
+    {
+      pairwise.col(i) = pairwise.col(i) / std::pow(accu(pow(
+        arma::abs(calculatedObjectives[front[i]]), dimension)), 1 / dimension);
+    }
+
+    while (remaining.size() > 0)
+    {
+      std::set<size_t>::iterator it;
+      value = value.fill(-1);
+      for (it = remaining.begin(); it != remaining.end(); it++)
+      {
+        value[*it] = DiversityScore<MatType>(selected, pairwise, *it);
+      }
+      size_t index = arma::index_max(value);
+      survivalScore[front[index]] = value[index];
+      selected.insert(index);
+      remaining.erase(index);
+    }
+  }
+
+  // Calculations for the other fronts.
+  else
+  {
+    for (size_t i = 0; i < front.size(); i++)
+    {
+      calculatedObjectives[front[i]] =
+          (calculatedObjectives[front[i]]) / normalize;
+      survivalScore[front[i]] =  1 / std::pow(accu(pow(abs(
+          calculatedObjectives[front[i]] - idealPoint), dimension)),
+              1 / dimension);
+    }
+  }
+}
+
+//! Comparator for survival score based sorting.
+template<typename MatType>
+inline bool AGEMOEA::SurvivalScoreOperator(
+    size_t idxP,
+    size_t idxQ,
+    const std::vector<size_t>& ranks,
+    const std::vector<typename MatType::elem_type>& survivalScore)
+{
+  if (ranks[idxP] < ranks[idxQ])
+    return true;
+  else if (ranks[idxP] == ranks[idxQ] && survivalScore[idxP] > survivalScore[idxQ])
+    return true;
+
+  return false;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp

@@ -30,7 +30,8 @@ namespace ens {
  * documentation on function types included with this distribution or on the
  * ensmallen website.
  */
-class AugLagrangian
+template<typename VecType = arma::vec> // TODO: remove for ensmallen 4.x
+class AugLagrangianType
 {
  public:
   /**
@@ -43,13 +44,13 @@ class AugLagrangian
    * @param maxIterations Maximum number of iterations of the Augmented
    *     Lagrangian algorithm.  0 indicates no maximum.
    */
-  AugLagrangian(const size_t maxIterations = 1000,
-                const double penaltyThresholdFactor = 0.25,
-                const double sigmaUpdateFactor = 10.0,
-                const L_BFGS& lbfgs = L_BFGS());
+  AugLagrangianType(const size_t maxIterations = 1000,
+                    const double penaltyThresholdFactor = 0.25,
+                    const double sigmaUpdateFactor = 10.0,
+                    const L_BFGS& lbfgs = L_BFGS());
 
   /**
-   * Optimize the function.  The value '1' is used for the initial value of each
+   * Optimize the function.  The value '0' is used for the initial value of each
    * Lagrange multiplier.  To set the Lagrange multipliers yourself, use the
    * other overload of Optimize().
    *
@@ -66,7 +67,8 @@ class AugLagrangian
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  typename std::enable_if<IsMatrixType<GradType>::value &&
+                          IsAllNonMatrix<CallbackTypes...>::value, bool>::type
   Optimize(LagrangianFunctionType& function,
            MatType& coordinates,
            CallbackTypes&&... callbacks);
@@ -75,9 +77,10 @@ class AugLagrangian
   template<typename LagrangianFunctionType,
            typename MatType,
            typename... CallbackTypes>
-  bool Optimize(LagrangianFunctionType& function,
-                MatType& coordinates,
-                CallbackTypes&&... callbacks)
+  typename std::enable_if<IsAllNonMatrix<CallbackTypes...>::value, bool>::type
+  Optimize(LagrangianFunctionType& function,
+           MatType& coordinates,
+           CallbackTypes&&... callbacks)
   {
     return Optimize<LagrangianFunctionType, MatType, MatType,
         CallbackTypes...>(function, coordinates,
@@ -96,29 +99,53 @@ class AugLagrangian
    * @tparam CallbackTypes Types of callback functions.
    * @param function The function to optimize.
    * @param coordinates Output matrix to store the optimized coordinates in.
-   * @param initLambda Vector of initial Lagrange multipliers.  Should have
-   *     length equal to the number of constraints.
-   * @param initSigma Initial penalty parameter.
+   * @param lambda Vector containing initial Lagrange multipliers.  Should have
+   *     length equal to the number of constraints.  This will be overwritten
+   *     with the Lagrange multipliers that are found during optimization.
+   * @param sigma Initial penalty parameter.  This will be overwritten with the
+   *     final penalty value used during optimization.
    * @param callbacks Callback functions.
    */
   template<typename LagrangianFunctionType,
            typename MatType,
+           typename InVecType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  [[deprecated("use Optimize() with non-const lambda/sigma instead")]]
+  typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
   Optimize(LagrangianFunctionType& function,
            MatType& coordinates,
-           const arma::vec& initLambda,
+           const InVecType& initLambda,
            const double initSigma,
+           CallbackTypes&&... callbacks)
+  {
+    deprecatedLambda = initLambda;
+    deprecatedSigma = initSigma;
+    return Optimize(function, coordinates, this->deprecatedLambda,
+        this->deprecatedSigma,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename InVecType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
+  Optimize(LagrangianFunctionType& function,
+           MatType& coordinates,
+           InVecType& lambda,
+           double& sigma,
            CallbackTypes&&... callbacks);
 
   //! Forward the MatType as GradType.
   template<typename LagrangianFunctionType,
            typename MatType,
            typename... CallbackTypes>
+  [[deprecated("use Optimize() with non-const lambda/sigma instead")]]
   bool Optimize(LagrangianFunctionType& function,
                 MatType& coordinates,
-                const arma::vec& initLambda,
+                const VecType& initLambda,
                 const double initSigma,
                 CallbackTypes&&... callbacks)
   {
@@ -127,20 +154,39 @@ class AugLagrangian
         std::forward<CallbackTypes>(callbacks)...);
   }
 
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename InVecType,
+           typename... CallbackTypes>
+  bool Optimize(LagrangianFunctionType& function,
+                MatType& coordinates,
+                InVecType& lambda,
+                double& sigma,
+                CallbackTypes&&... callbacks)
+  {
+    return Optimize<LagrangianFunctionType, MatType, InVecType, MatType,
+        CallbackTypes...>(function, coordinates, lambda, sigma,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
   //! Get the L-BFGS object used for the actual optimization.
   const L_BFGS& LBFGS() const { return lbfgs; }
   //! Modify the L-BFGS object used for the actual optimization.
   L_BFGS& LBFGS() { return lbfgs; }
 
   //! Get the Lagrange multipliers.
-  const arma::vec& Lambda() const { return lambda; }
+  [[deprecated("use Optimize() with lambda/sigma parameters instead")]]
+  const VecType& Lambda() const { return deprecatedLambda; }
   //! Modify the Lagrange multipliers (i.e. set them before optimization).
-  arma::vec& Lambda() { return lambda; }
+  [[deprecated("use Optimize() with lambda/sigma parameters instead")]]
+  VecType& Lambda() { return deprecatedLambda; }
 
   //! Get the penalty parameter.
-  double Sigma() const { return sigma; }
+  [[deprecated("use Optimize() with lambda/sigma parameters instead")]]
+  double Sigma() const { return deprecatedSigma; }
   //! Modify the penalty parameter.
-  double& Sigma() { return sigma; }
+  [[deprecated("use Optimize() with lambda/sigma parameters instead")]]
+  double& Sigma() { return deprecatedSigma; }
 
   //! Get the maximum iterations
   size_t MaxIterations() const { return maxIterations; }
@@ -173,11 +219,11 @@ class AugLagrangian
   //! Controls early termination of the optimization process.
   bool terminate;
 
+  // NOTE: these will be removed in ensmallen 4.x!
   //! Lagrange multipliers.
-  arma::vec lambda;
-
+  VecType deprecatedLambda;
   //! Penalty parameter.
-  double sigma;
+  double deprecatedSigma;
 
   /**
    * Internal optimization function: given an initialized AugLagrangianFunction,
@@ -185,27 +231,32 @@ class AugLagrangian
    */
   template<typename LagrangianFunctionType,
            typename MatType,
+           typename InVecType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
-  Optimize(AugLagrangianFunction<LagrangianFunctionType>& augfunc,
+  typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
+  Optimize(AugLagrangianFunction<LagrangianFunctionType, InVecType>& augfunc,
            MatType& coordinates,
            CallbackTypes&&... callbacks);
 
   //! Forward the MatType as GradType.
   template<typename LagrangianFunctionType,
            typename MatType,
+           typename InVecType,
            typename... CallbackTypes>
-  bool Optimize(AugLagrangianFunction<LagrangianFunctionType>& function,
-                MatType& coordinates,
-                CallbackTypes&&... callbacks)
+  bool Optimize(
+      AugLagrangianFunction<LagrangianFunctionType, InVecType>& function,
+      MatType& coordinates,
+      CallbackTypes&&... callbacks)
   {
-    return Optimize<LagrangianFunctionType, MatType, MatType,
+    return Optimize<LagrangianFunctionType, MatType, InVecType, MatType,
         CallbackTypes...>(function, coordinates,
         std::forward<CallbackTypes>(callbacks)...);
   }
 };
 
+using AugLagrangian = AugLagrangianType<arma::vec>;
+
 } // namespace ens
 
 #include "aug_lagrangian_impl.hpp"

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_function.hpp

@@ -31,19 +31,10 @@ namespace ens {
  *
  * @tparam LagrangianFunction Lagrangian function to be used.
  */
-template<typename LagrangianFunction>
+template<typename LagrangianFunction, typename VecType>
 class AugLagrangianFunction
 {
  public:
-  /**
-   * Initialize the AugLagrangianFunction, but don't set the Lagrange
-   * multipliers or penalty parameters yet.  Make sure you set the Lagrange
-   * multipliers before you use this...
-   *
-   * @param function Lagrangian function.
-   */
-  AugLagrangianFunction(LagrangianFunction& function);
-
   /**
    * Initialize the AugLagrangianFunction with the given LagrangianFunction,
    * Lagrange multipliers, and initial penalty parameter.
@@ -53,8 +44,8 @@ class AugLagrangianFunction
    * @param sigma Initial penalty parameter.
    */
   AugLagrangianFunction(LagrangianFunction& function,
-                        const arma::vec& lambda,
-                        const double sigma);
+                        VecType& lambda,
+                        double& sigma);
   /**
    * Evaluate the objective function of the Augmented Lagrangian function, which
    * is the standard Lagrangian function evaluation plus a penalty term, which
@@ -81,17 +72,12 @@ class AugLagrangianFunction
    *
    * @return Initial point.
    */
-  template<typename MatType = arma::mat>
+  template<typename MatType>
   const MatType& GetInitialPoint() const;
 
-  //! Get the Lagrange multipliers.
-  const arma::vec& Lambda() const { return lambda; }
-  //! Modify the Lagrange multipliers.
-  arma::vec& Lambda() { return lambda; }
-
-  //! Get sigma (the penalty parameter).
-  double Sigma() const { return sigma; }
-  //! Modify sigma (the penalty parameter).
+  // Get the Lagrange multipliers.
+  VecType& Lambda() { return lambda; }
+  // Get the penalty parameter.
   double& Sigma() { return sigma; }
 
   //! Get the Lagrangian function.
@@ -104,9 +90,9 @@ class AugLagrangianFunction
   LagrangianFunction& function;
 
   //! The Lagrange multipliers.
-  arma::vec lambda;
+  VecType& lambda;
   //! The penalty parameter.
-  double sigma;
+  double& sigma;
 };
 
 } // namespace ens

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_function_impl.hpp

@@ -20,23 +20,11 @@
 namespace ens {
 
 // Initialize the AugLagrangianFunction.
-template<typename LagrangianFunction>
-AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
-    LagrangianFunction& function) :
-    function(function),
-    lambda(function.NumConstraints()),
-    sigma(10)
-{
-  // Initialize lambda vector to all zeroes.
-  lambda.zeros();
-}
-
-// Initialize the AugLagrangianFunction.
-template<typename LagrangianFunction>
-AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
+template<typename LagrangianFunction, typename VecType>
+AugLagrangianFunction<LagrangianFunction, VecType>::AugLagrangianFunction(
     LagrangianFunction& function,
-    const arma::vec& lambda,
-    const double sigma) :
+    VecType& lambda,
+    double& sigma) :
     function(function),
     lambda(lambda),
     sigma(sigma)
@@ -45,9 +33,10 @@ AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
 }
 
 // Evaluate the AugLagrangianFunction at the given coordinates.
-template<typename LagrangianFunction>
+template<typename LagrangianFunction, typename VecType>
 template<typename MatType>
-typename MatType::elem_type AugLagrangianFunction<LagrangianFunction>::Evaluate(
+typename MatType::elem_type
+AugLagrangianFunction<LagrangianFunction, VecType>::Evaluate(
     const MatType& coordinates) const
 {
   // The augmented Lagrangian is evaluated as
@@ -63,20 +52,22 @@ typename MatType::elem_type AugLagrangianFunction<LagrangianFunction>::Evaluate(
   {
     ElemType constraint = function.EvaluateConstraint(i, coordinates);
 
-    objective += (-lambda[i] * constraint) +
-        sigma * std::pow(constraint, 2) / 2;
+    objective += (-ElemType(lambda[i]) * constraint) +
+        ElemType(sigma) * std::pow(constraint, ElemType(2)) / 2;
   }
 
   return objective;
 }
 
 // Evaluate the gradient of the AugLagrangianFunction at the given coordinates.
-template<typename LagrangianFunction>
+template<typename LagrangianFunction, typename VecType>
 template<typename MatType, typename GradType>
-void AugLagrangianFunction<LagrangianFunction>::Gradient(
+void AugLagrangianFunction<LagrangianFunction, VecType>::Gradient(
     const MatType& coordinates,
     GradType& gradient) const
 {
+  typedef typename MatType::elem_type ElemType;
+
   // The augmented Lagrangian's gradient is evaluted as
   // f'(x) + {(-lambda_i + sigma * c_i(x)) * c'_i(x)} for all constraints
   gradient.zeros();
@@ -89,16 +80,17 @@ void AugLagrangianFunction<LagrangianFunction>::Gradient(
 
     // Now calculate scaling factor and add to existing gradient.
     GradType tmpGradient;
-    tmpGradient = (-lambda[i] + sigma *
+    tmpGradient = (ElemType(-lambda[i]) + ElemType(sigma) *
         function.EvaluateConstraint(i, coordinates)) * constraintGradient;
     gradient += tmpGradient;
   }
 }
 
 // Get the initial point.
-template<typename LagrangianFunction>
+template<typename LagrangianFunction, typename VecType>
 template<typename MatType>
-const MatType& AugLagrangianFunction<LagrangianFunction>::GetInitialPoint()
+const MatType&
+AugLagrangianFunction<LagrangianFunction, VecType>::GetInitialPoint()
     const
 {
   return function.template GetInitialPoint<MatType>();

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_impl.hpp

@@ -19,70 +19,90 @@
 
 namespace ens {
 
-inline AugLagrangian::AugLagrangian(const size_t maxIterations,
-                                    const double penaltyThresholdFactor,
-                                    const double sigmaUpdateFactor,
-                                    const L_BFGS& lbfgs) :
+template<typename VecType>
+inline AugLagrangianType<VecType>::AugLagrangianType(
+    const size_t maxIterations,
+    const double penaltyThresholdFactor,
+    const double sigmaUpdateFactor,
+    const L_BFGS& lbfgs) :
     maxIterations(maxIterations),
     penaltyThresholdFactor(penaltyThresholdFactor),
     sigmaUpdateFactor(sigmaUpdateFactor),
     lbfgs(lbfgs),
     terminate(false),
-    sigma(0.0)
+    deprecatedSigma(0.0)
 {
 }
 
+template<typename VecType>
 template<typename LagrangianFunctionType,
          typename MatType,
+         typename InVecType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value, bool>::type
-AugLagrangian::Optimize(LagrangianFunctionType& function,
-                        MatType& coordinates,
-                        const arma::vec& initLambda,
-                        const double initSigma,
-                        CallbackTypes&&... callbacks)
+typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
+AugLagrangianType<VecType>::Optimize(
+    LagrangianFunctionType& function,
+    MatType& coordinates,
+    InVecType& lambda,
+    double& sigma,
+    CallbackTypes&&... callbacks)
 {
-  lambda = initLambda;
-  sigma = initSigma;
-
-  AugLagrangianFunction<LagrangianFunctionType> augfunc(function,
-      lambda, sigma);
+  AugLagrangianFunction<LagrangianFunctionType, InVecType> augfunc(
+      function, lambda, sigma);
 
   return Optimize(augfunc, coordinates, callbacks...);
 }
 
+template<typename VecType>
 template<typename LagrangianFunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value, bool>::type
-AugLagrangian::Optimize(LagrangianFunctionType& function,
-                        MatType& coordinates,
-                        CallbackTypes&&... callbacks)
+typename std::enable_if<IsMatrixType<GradType>::value &&
+                        IsAllNonMatrix<CallbackTypes...>::value, bool>::type
+AugLagrangianType<VecType>::Optimize(LagrangianFunctionType& function,
+                                     MatType& coordinates,
+                                     CallbackTypes&&... callbacks)
 {
+  typedef typename ForwardType<MatType>::bvec InVecType;
+
   // If the user did not specify the right size for sigma and lambda, we will
   // use defaults.
-  if (!lambda.is_empty())
+  // TODO: remove this when ensmallen 4.x is released!
+  if (!deprecatedLambda.is_empty())
   {
-    AugLagrangianFunction<LagrangianFunctionType> augfunc(function, lambda,
-        sigma);
-    return Optimize(augfunc, coordinates, callbacks...);
+    InVecType lambda(conv_to<InVecType>::from(deprecatedLambda));
+
+    AugLagrangianFunction<LagrangianFunctionType, InVecType> augfunc(function,
+        lambda, deprecatedSigma);
+    const bool result = Optimize(augfunc, coordinates, callbacks...);
+    deprecatedLambda = conv_to<VecType>::from(lambda);
+
+    return result;
   }
   else
   {
-    AugLagrangianFunction<LagrangianFunctionType> augfunc(function);
+    // Use default values.
+    InVecType lambda(function.NumConstraints());
+    lambda.zeros();
+    double sigma = 10;
+
+    AugLagrangianFunction<LagrangianFunctionType, InVecType> augfunc(
+        function, lambda, sigma);
     return Optimize(augfunc, coordinates, callbacks...);
   }
 }
 
+template<typename VecType>
 template<typename LagrangianFunctionType,
          typename MatType,
+         typename InVecType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value, bool>::type
-AugLagrangian::Optimize(
-    AugLagrangianFunction<LagrangianFunctionType>& augfunc,
+typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
+AugLagrangianType<VecType>::Optimize(
+    AugLagrangianFunction<LagrangianFunctionType, InVecType>& augfunc,
     MatType& coordinatesIn,
     CallbackTypes&&... callbacks)
 {
@@ -110,15 +130,16 @@ AugLagrangian::Optimize(
 
   // Convergence tolerance---depends on the epsilon of the type we are using for
   // optimization.
-  ElemType tolerance = 1e3 * std::numeric_limits<ElemType>::epsilon();
+  ElemType tolerance = 1000 * std::numeric_limits<ElemType>::epsilon();
 
   // Then, calculate the current penalty.
   ElemType penalty = 0;
   for (size_t i = 0; i < function.NumConstraints(); i++)
   {
-    const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates), 2);
-    Callback::EvaluateConstraint(*this, function, coordinates, i, p,
-          callbacks...);
+    const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates),
+        ElemType(2));
+    terminate |= Callback::EvaluateConstraint(*this, function, coordinates, i,
+        p, callbacks...);
 
     penalty += p;
   }
@@ -129,21 +150,19 @@ AugLagrangian::Optimize(
   // The odd comparison allows user to pass maxIterations = 0 (i.e. no limit on
   // number of iterations).
   size_t it;
-  terminate |= Callback::BeginOptimization(*this, function, coordinates,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, coordinates, callbacks...);
   for (it = 0; it != (maxIterations - 1) && !terminate; it++)
   {
     Info << "AugLagrangian on iteration " << it
         << ", starting with objective "  << lastObjective << "." << std::endl;
 
     if (!lbfgs.Optimize(augfunc, coordinates, callbacks...))
-      Info << "L-BFGS reported an error during optimization."
-          << std::endl;
-    Info << "Done with L-BFGS: " << coordinates << "\n";
+      Info << "L-BFGS reported an error during optimization." << std::endl;
+    Info << "Done with L-BFGS." << std::endl;
 
     const ElemType objective = function.Evaluate(coordinates);
 
-    Callback::Evaluate(*this, function, coordinates, objective,
+    terminate |= Callback::Evaluate(*this, function, coordinates, objective,
         callbacks...);
 
     // Check if we are done with the entire optimization (the threshold we are
@@ -151,9 +170,6 @@ AugLagrangian::Optimize(
     if (std::abs(lastObjective - objective) < tolerance &&
         augfunc.Sigma() > 500000)
     {
-      lambda = std::move(augfunc.Lambda());
-      sigma = augfunc.Sigma();
-
       Callback::EndOptimization(*this, function, coordinates, callbacks...);
       return true;
     }
@@ -169,15 +185,18 @@ AugLagrangian::Optimize(
     for (size_t i = 0; i < function.NumConstraints(); i++)
     {
       const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates),
-          2);
-      Callback::EvaluateConstraint(*this, function, coordinates, i, p,
-          callbacks...);
+          ElemType(2));
+      terminate |= Callback::EvaluateConstraint(*this, function, coordinates, i,
+          p, callbacks...);
 
       penalty += p;
     }
 
-    Info << "Penalty is " << penalty << " (threshold "
-        << penaltyThreshold << ")." << std::endl;
+    Info << "Penalty is " << penalty << " (threshold " << penaltyThreshold
+        << ")." << std::endl;
+
+    if (terminate)
+      break;
 
     if (penalty < penaltyThreshold) // We update lambda.
     {
@@ -186,15 +205,15 @@ AugLagrangian::Optimize(
       for (size_t i = 0; i < function.NumConstraints(); i++)
       {
         const ElemType p = function.EvaluateConstraint(i, coordinates);
-        Callback::EvaluateConstraint(*this, function, coordinates, i, p,
-          callbacks...);
+        terminate |= Callback::EvaluateConstraint(*this, function, coordinates,
+            i, p, callbacks...);
 
-        augfunc.Lambda()[i] -= augfunc.Sigma() * p;
+        augfunc.Lambda()[i] -= ElemType(augfunc.Sigma()) * p;
       }
 
       // We also update the penalty threshold to be a factor of the current
       // penalty.
-      penaltyThreshold = penaltyThresholdFactor * penalty;
+      penaltyThreshold = ElemType(penaltyThresholdFactor) * penalty;
       Info << "Lagrange multiplier estimates updated." << std::endl;
     }
     else
@@ -207,7 +226,7 @@ AugLagrangian::Optimize(
         Warn << "AugLagrangian::Optimize(): sigma too large for element type; "
             << "terminating." << std::endl;
         Callback::EndOptimization(*this, function, coordinates, callbacks...);
-        return false;
+        return true;
       }
     }
 

include/ensmallen_bits/bigbatch_sgd/adaptive_stepsize.hpp

@@ -69,6 +69,8 @@ class AdaptiveStepsize
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     // Create the instantiated object.
     Policy(AdaptiveStepsize& parent) : parent(parent) { }
 
@@ -104,7 +106,7 @@ class AdaptiveStepsize
           backtrackingBatchSize);
 
       // Update the iterate.
-      iterate -= stepSize * gradient;
+      iterate -= ElemType(stepSize) * gradient;
 
       // Update Gradient & calculate curvature of quadratic approximation.
       GradType functionGradient(iterate.n_rows, iterate.n_cols);
@@ -132,8 +134,8 @@ class AdaptiveStepsize
         delta0 = delta1 + (functionGradient - delta1) / k;
 
         // Compute sample variance.
-        vB += arma::norm(functionGradient - delta1, 2.0) *
-            arma::norm(functionGradient - delta0, 2.0);
+        vB += norm(functionGradient - delta1, 2.0) *
+            norm(functionGradient - delta0, 2.0);
 
         delta1 = delta0;
         gradient += functionGradient;
@@ -145,13 +147,13 @@ class AdaptiveStepsize
 
       // Update sample variance & norm of the gradient.
       sampleVariance = vB;
-      gradientNorm = std::pow(arma::norm(gradient / backtrackingBatchSize, 2),
+      gradientNorm = std::pow(norm(gradient / backtrackingBatchSize, 2),
           2.0);
 
       // Compute curvature.
-      double v = arma::trace(arma::trans(iterate - iteratePrev) *
+      double v = trace(trans(iterate - iteratePrev) *
           (gradient - gradPrevIterate)) /
-          std::pow(arma::norm(iterate - iteratePrev, 2), 2.0);
+          std::pow(norm(iterate - iteratePrev, 2), 2.0);
 
       // Update previous iterate.
       iteratePrev = iterate;
@@ -205,12 +207,10 @@ class AdaptiveStepsize
                       const size_t offset,
                       const size_t backtrackingBatchSize)
     {
-      typedef typename MatType::elem_type ElemType;
-
       ElemType overallObjective = function.Evaluate(iterate,
           offset, backtrackingBatchSize);
 
-      MatType iterateUpdate = iterate - (stepSize * gradient);
+      MatType iterateUpdate = iterate - (ElemType(stepSize) * gradient);
       ElemType overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
           backtrackingBatchSize);
 
@@ -220,7 +220,7 @@ class AdaptiveStepsize
       {
         stepSize *= parent.backtrackStepSize;
 
-        iterateUpdate = iterate - (stepSize * gradient);
+        iterateUpdate = iterate - (ElemType(stepSize) * gradient);
         overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
             backtrackingBatchSize);
       }

include/ensmallen_bits/bigbatch_sgd/backtracking_line_search.hpp

@@ -60,6 +60,8 @@ class BacktrackingLineSearch
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     // Instantiate the policy with the given parent.
     Policy(BacktrackingLineSearch& parent) : parent(parent) { }
 
@@ -94,12 +96,10 @@ class BacktrackingLineSearch
       if (reset)
         stepSize *= 2;
 
-      typedef typename MatType::elem_type ElemType;
-
       ElemType overallObjective = function.Evaluate(iterate, offset,
           backtrackingBatchSize);
 
-      MatType iterateUpdate = iterate - (stepSize * gradient);
+      MatType iterateUpdate = iterate - (ElemType(stepSize) * gradient);
       ElemType overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
           backtrackingBatchSize);
 
@@ -109,7 +109,7 @@ class BacktrackingLineSearch
       {
         stepSize /= 2;
 
-        iterateUpdate = iterate - (stepSize * gradient);
+        iterateUpdate = iterate - (ElemType(stepSize) * gradient);
         overallObjectiveUpdate = function.Evaluate(iterateUpdate,
           offset, backtrackingBatchSize);
       }

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp

@@ -125,7 +125,7 @@ class BigBatchSGD
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd_impl.hpp

@@ -50,8 +50,8 @@ template<typename SeparableFunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 BigBatchSGD<UpdatePolicyType>::Optimize(
     SeparableFunctionType& function,
     MatType& iterateIn,
@@ -104,7 +104,7 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
   BaseGradType functionGradient(iterate.n_rows, iterate.n_cols);
   const size_t actualMaxIterations = (maxIterations == 0) ?
       std::numeric_limits<size_t>::max() : maxIterations;
-  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
   for (size_t i = 0; i < actualMaxIterations && !terminate;
       /* incrementing done manually */)
   {
@@ -137,13 +137,13 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
       delta0 = delta1 + (functionGradient - delta1) / k;
 
       // Compute sample variance.
-      vB += arma::norm(functionGradient - delta1, 2.0) *
-          arma::norm(functionGradient - delta0, 2.0);
+      vB += norm(functionGradient - delta1, 2.0) *
+          norm(functionGradient - delta0, 2.0);
 
       delta1 = delta0;
       gradient += functionGradient;
     }
-    double gB = std::pow(arma::norm(gradient / effectiveBatchSize, 2), 2.0);
+    double gB = std::pow(norm(gradient / effectiveBatchSize, 2), 2.0);
 
     // Reset the batch size update process counter.
     reset = false;
@@ -174,13 +174,13 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
           delta0 = delta1 + (functionGradient - delta1) / (k + 1);
 
           // Compute sample variance.
-          vB += arma::norm(functionGradient - delta1, 2.0) *
-              arma::norm(functionGradient - delta0, 2.0);
+          vB += norm(functionGradient - delta1, 2.0) *
+              norm(functionGradient - delta0, 2.0);
 
           delta1 = delta0;
           gradient += functionGradient;
         }
-        gB = std::pow(arma::norm(gradient / (batchSize + batchOffset), 2), 2.0);
+        gB = std::pow(norm(gradient / (batchSize + batchOffset), 2), 2.0);
 
         // Update the batchSize.
         batchSize += batchOffset;
@@ -191,12 +191,15 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
       }
     }
 
+    if (terminate)
+      break;
+
     instUpdatePolicy.As<InstUpdatePolicyType>().Update(f, stepSize, iterate,
         gradient, gB, vB, currentFunction, batchSize, effectiveBatchSize,
         reset);
 
     // Update the iterate.
-    iterate -= stepSize * gradient;
+    iterate -= ElemType(stepSize) * gradient;
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
 
     const ElemType objective = f.Evaluate(iterate, currentFunction,
@@ -229,9 +232,7 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
         return overallObjective;
       }
 
-      if (std::abs(lastObjective - overallObjective) < tolerance ||
-          Callback::BeginEpoch(*this, f, iterate, epoch, overallObjective,
-              callbacks...))
+      if (std::abs(lastObjective - overallObjective) < tolerance)
       {
         Info << "Big-batch SGD: minimized within tolerance " << tolerance
             << "; terminating optimization." << std::endl;
@@ -240,18 +241,27 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
         return overallObjective;
       }
 
-      // Reset the counter variables.
-      lastObjective = overallObjective;
-      overallObjective = 0;
-      currentFunction = 0;
+      terminate |= Callback::BeginEpoch(*this, f, iterate, epoch,
+          overallObjective, callbacks...);
+
+      // Reset the counter variables if we will continue.
+      if (i != actualMaxIterations)
+      {
+        lastObjective = overallObjective;
+        overallObjective = 0;
+        currentFunction = 0;
+      }
 
       if (shuffle) // Determine order of visitation.
         f.Shuffle();
     }
   }
 
-  Info << "Big-batch SGD: maximum iterations (" << maxIterations << ") "
-      << "reached; terminating optimization." << std::endl;
+  if (!terminate)
+  {
+    Info << "Big-batch SGD: maximum iterations (" << maxIterations << ") "
+        << "reached; terminating optimization." << std::endl;
+  }
 
   // Calculate final objective if exactObjective is set to true.
   if (exactObjective)
@@ -263,7 +273,9 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
       const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
       overallObjective += objective;
 
-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      // The optimization is finished, so we don't need to care what the
+      // callback returns.
+      (void) Callback::Evaluate(*this, f, iterate, objective, callbacks...);
     }
   }
 

include/ensmallen_bits/callbacks/callbacks.hpp

@@ -20,32 +20,35 @@ namespace ens {
  * Callbacks are a set of functions that can be applied at given stages of the
  * optimization process. The following callbacks are available:
  *
- * - Evaluate(optimizer, function, coordinates, objective):
+ * - bool Evaluate(optimizer, function, coordinates, objective):
  *   called after any call to Evaluate().
  *
- * - StepTaken(optimizer, function, coordinates):
+ * - bool StepTaken(optimizer, function, coordinates):
  *   called after any step is taken.
  *
- * - Gradient(optimizer, function, coordinates, gradient):
+ * - bool Gradient(optimizer, function, coordinates, gradient):
  *   called whenever the gradient is computed.
  *
- * - BeginEpoch(optimizer, function, coordinates, epoch, objective):
+ * - bool BeginEpoch(optimizer, function, coordinates, epoch, objective):
  *   called at the beginning of a pass over the data. The objective may be
  *   exact or an estimate depending on exactObjective's value.
  *
- * - EvaluateConstraint(optimizer, function, coordinates, constraint,
- *                      constraintValue):
+ * - bool EvaluateConstraint(optimizer, function, coordinates, constraint,
+ *                           constraintValue):
  *   called after any call to EvaluateConstraint().
  *
- * - GradientConstraint(optimizer, function, coordinates, constraint,
- *                      constraintGradient):
+ * - bool GradientConstraint(optimizer, function, coordinates, constraint,
+ *                           constraintGradient):
  *   called after any call to GradientConstraint().
  *
- * - BeginOptimization(optimizer, function, coordinates):
+ * - void BeginOptimization(optimizer, function, coordinates):
  *   called at the beginning of the optimization.
  *
- * - EndOptimization(optimizer, function, coordinates):
+ * - void EndOptimization(optimizer, function, coordinates):
  *   called at the end of the optimization.
+ *
+ * If true is returned to any of the bool-type callbacks, the optimization will
+ * be terminated before any more steps are taken.
  */
 class Callback
 {
@@ -64,14 +67,14 @@ class Callback
            typename MatType>
   static typename std::enable_if<
       callbacks::traits::HasBeginOptimizationSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
-      bool>::type
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      void>::type
   BeginOptimizationFunction(CallbackType& callback,
                             OptimizerType& optimizer,
                             FunctionType& function,
                             MatType& coordinates)
   {
-    return const_cast<CallbackType&>(callback).BeginOptimization(optimizer,
+    (void) const_cast<CallbackType&>(callback).BeginOptimization(optimizer,
         function, coordinates);
   }
 
@@ -80,32 +83,14 @@ class Callback
            typename FunctionType,
            typename MatType>
   static typename std::enable_if<
-      callbacks::traits::HasBeginOptimizationSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
-      bool>::type
-  BeginOptimizationFunction(CallbackType& callback,
-                            OptimizerType& optimizer,
-                            FunctionType& function,
-                            MatType& coordinates)
-  {
-    const_cast<CallbackType&>(callback).BeginOptimization(optimizer, function,
-        coordinates);
-    return false;
-  }
-
-  template<typename CallbackType,
-           typename OptimizerType,
-           typename FunctionType,
-           typename MatType>
-  static typename std::enable_if<
-      callbacks::traits::HasBeginOptimizationSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
-      bool>::type
+      !callbacks::traits::HasBeginOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      void>::type
   BeginOptimizationFunction(CallbackType& /* callback */,
                             OptimizerType& /* optimizer */,
                             FunctionType& /* function */,
                             MatType& /* coordinates */)
-  { return false; }
+  { /* Nothing to do. */ }
 
   /**
    * Iterate over the callbacks and invoke the BeginOptimization() callback if
@@ -119,18 +104,28 @@ class Callback
   template<typename OptimizerType,
            typename FunctionType,
            typename MatType,
+           typename CallbackType,
            typename... CallbackTypes>
-  static bool BeginOptimization(OptimizerType& optimizer,
+  static void BeginOptimization(OptimizerType& optimizer,
                                 FunctionType& function,
                                 MatType& coordinates,
-                                CallbackTypes&... callbacks)
+                                CallbackType& callback,
+                                CallbackTypes&... otherCallbacks)
   {
-    // This will return immediately once a callback returns true.
-    bool result = false;
-    (void)std::initializer_list<bool>{ result =
-        result || Callback::BeginOptimizationFunction(callbacks, optimizer,
-            function, coordinates)... };
-     return result;
+    Callback::BeginOptimizationFunction(callback, optimizer, function,
+        coordinates);
+    Callback::BeginOptimization(optimizer, function, coordinates,
+        otherCallbacks...);
+  }
+
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static void BeginOptimization(OptimizerType& /* optimizer */,
+                                FunctionType& /* function */,
+                                MatType& /* coordinates */)
+  {
+    // Base case... no callbacks left.  Nothing to do.
   }
 
   /**
@@ -147,28 +142,29 @@ class Callback
            typename MatType>
   static typename std::enable_if<callbacks::traits::HasEndOptimizationSignature<
       CallbackType, OptimizerType, FunctionType, MatType>::value,
-      bool>::type
+      void>::type
   EndOptimizationFunction(CallbackType& callback,
                           OptimizerType& optimizer,
                           FunctionType& function,
                           MatType& coordinates)
   {
-    return (const_cast<CallbackType&>(callback).EndOptimization(
-        optimizer, function, coordinates), false);
+    (void) const_cast<CallbackType&>(callback).EndOptimization( optimizer,
+        function, coordinates);
   }
 
   template<typename CallbackType,
            typename OptimizerType,
            typename FunctionType,
            typename MatType>
-  static typename std::enable_if<!callbacks::traits::HasEndOptimizationSignature<
+  static typename std::enable_if<
+      !callbacks::traits::HasEndOptimizationSignature<
       CallbackType, OptimizerType, FunctionType, MatType>::value,
-      bool>::type
+      void>::type
   EndOptimizationFunction(CallbackType& /* callback */,
                           OptimizerType& /* optimizer */,
                           FunctionType& /* function */,
                           MatType& /* coordinates */)
-  { return false; }
+  { /* Nothing to do. */ }
 
   /**
    * Iterate over the callbacks and invoke the EndOptimization() callback if it
@@ -182,18 +178,28 @@ class Callback
   template<typename OptimizerType,
            typename FunctionType,
            typename MatType,
+           typename CallbackType,
            typename... CallbackTypes>
-  static bool EndOptimization(OptimizerType& optimizer,
+  static void EndOptimization(OptimizerType& optimizer,
                               FunctionType& function,
                               MatType& coordinates,
-                              CallbackTypes&... callbacks)
+                              CallbackType& callback,
+                              CallbackTypes&... otherCallbacks)
   {
-    // This will return immediately once a callback returns true.
-    bool result = false;
-    (void)std::initializer_list<bool>{ result =
-        result || Callback::EndOptimizationFunction(callbacks, optimizer,
-            function, coordinates)... };
-     return result;
+    Callback::EndOptimizationFunction(callback, optimizer, function,
+        coordinates);
+    Callback::EndOptimization(optimizer, function, coordinates,
+        otherCallbacks...);
+  }
+
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static void EndOptimization(OptimizerType& /* optimizer */,
+                              FunctionType& /* function */,
+                              MatType& /* coordinates */)
+  {
+    // Base case... no callbacks left.  Nothing to do.
   }
 
   /**
@@ -210,7 +216,7 @@ class Callback
            typename FunctionType,
            typename MatType>
   static typename std::enable_if<callbacks::traits::HasEvaluateSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
       bool>::type
   EvaluateFunction(CallbackType& callback,
                    OptimizerType& optimizer,
@@ -218,16 +224,34 @@ class Callback
                    const MatType& coordinates,
                    const double objective)
   {
-    return (const_cast<CallbackType&>(callback).Evaluate(
-        optimizer, function, coordinates, objective), false);
+    return const_cast<CallbackType&>(callback).Evaluate(optimizer, function,
+        coordinates, objective);
   }
 
   template<typename CallbackType,
            typename OptimizerType,
            typename FunctionType,
            typename MatType>
-  static typename std::enable_if<!callbacks::traits::HasEvaluateSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value,
+  static typename std::enable_if<callbacks::traits::HasEvaluateSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
+      bool>::type
+  EvaluateFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   const double objective)
+  {
+    const_cast<CallbackType&>(callback).Evaluate(optimizer, function,
+        coordinates, objective);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEvaluateSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
       bool>::type
   EvaluateFunction(CallbackType& /* callback */,
                    OptimizerType& /* optimizer */,
@@ -280,7 +304,7 @@ class Callback
            typename MatType>
   static typename std::enable_if<
       callbacks::traits::HasEvaluateConstraintSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
       bool>::type
   EvaluateConstraintFunction(CallbackType& callback,
                              OptimizerType& optimizer,
@@ -289,8 +313,8 @@ class Callback
                              const size_t constraint,
                              const double constraintValue)
   {
-    return (const_cast<CallbackType&>(callback).EvaluateConstraint(
-        optimizer, function, coordinates, constraint, constraintValue), false);
+    return const_cast<CallbackType&>(callback).EvaluateConstraint(
+        optimizer, function, coordinates, constraint, constraintValue);
   }
 
   template<typename CallbackType,
@@ -298,8 +322,28 @@ class Callback
            typename FunctionType,
            typename MatType>
   static typename std::enable_if<
-      !callbacks::traits::HasEvaluateConstraintSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      callbacks::traits::HasEvaluateConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
+      bool>::type
+  EvaluateConstraintFunction(CallbackType& callback,
+                             OptimizerType& optimizer,
+                             FunctionType& function,
+                             const MatType& coordinates,
+                             const size_t constraint,
+                             const double constraintValue)
+  {
+    const_cast<CallbackType&>(callback).EvaluateConstraint(
+        optimizer, function, coordinates, constraint, constraintValue);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasEvaluateConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
       bool>::type
   EvaluateConstraintFunction(CallbackType& /* callback */,
                              OptimizerType& /* optimizer */,
@@ -356,7 +400,7 @@ class Callback
            typename MatType,
            typename GradType>
   static typename std::enable_if<callbacks::traits::HasGradientSignature<
-      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasBool,
       bool>::type
   GradientFunction(CallbackType& callback,
                    OptimizerType& optimizer,
@@ -364,8 +408,8 @@ class Callback
                    const MatType& coordinates,
                    GradType& gradient)
   {
-    return (const_cast<CallbackType&>(callback).Gradient(
-        optimizer, function, coordinates, gradient), false);
+    return const_cast<CallbackType&>(callback).Gradient(optimizer, function,
+        coordinates, gradient);
   }
 
   template<typename CallbackType,
@@ -373,8 +417,27 @@ class Callback
            typename FunctionType,
            typename MatType,
            typename GradType>
-  static typename std::enable_if<!callbacks::traits::HasGradientSignature<
-      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+  static typename std::enable_if<callbacks::traits::HasGradientSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasVoid,
+      bool>::type
+  GradientFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   GradType& gradient)
+  {
+    const_cast<CallbackType&>(callback).Gradient(
+        optimizer, function, coordinates, gradient);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<callbacks::traits::HasGradientSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasNone,
       bool>::type
   GradientFunction(CallbackType& /* callback */,
                    OptimizerType& /* optimizer */,
@@ -427,7 +490,7 @@ class Callback
            typename GradType>
   static typename std::enable_if<
       callbacks::traits::HasGradientConstraintSignature<
-      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasBool,
       bool>::type
   GradientConstraintFunction(CallbackType& callback,
                              OptimizerType& optimizer,
@@ -436,8 +499,8 @@ class Callback
                              const size_t constraint,
                              GradType& gradient)
   {
-    return (const_cast<CallbackType&>(callback).GradientConstraint(
-        optimizer, function, coordinates, constraint, gradient), false);
+    return const_cast<CallbackType&>(callback).GradientConstraint(optimizer,
+        function, coordinates, constraint, gradient);
   }
 
   template<typename CallbackType,
@@ -446,8 +509,29 @@ class Callback
            typename MatType,
            typename GradType>
   static typename std::enable_if<
-      !callbacks::traits::HasGradientConstraintSignature<
-      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      callbacks::traits::HasGradientConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasVoid,
+      bool>::type
+  GradientConstraintFunction(CallbackType& callback,
+                             OptimizerType& optimizer,
+                             FunctionType& function,
+                             const MatType& coordinates,
+                             const size_t constraint,
+                             GradType& gradient)
+  {
+    const_cast<CallbackType&>(callback).GradientConstraint(
+        optimizer, function, coordinates, constraint, gradient);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<
+      callbacks::traits::HasGradientConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::hasNone,
       bool>::type
   GradientConstraintFunction(CallbackType& /* callback */,
                              OptimizerType& /* optimizer */,
@@ -539,7 +623,7 @@ class Callback
            typename FunctionType,
            typename MatType>
   static typename std::enable_if<callbacks::traits::HasBeginEpochSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value, bool>::type
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool, bool>::type
   BeginEpochFunction(CallbackType& callback,
                      OptimizerType& optimizer,
                      FunctionType& function,
@@ -547,16 +631,34 @@ class Callback
                      const size_t epoch,
                      const double objective)
   {
-    return (const_cast<CallbackType&>(callback).BeginEpoch(
-        optimizer, function, coordinates, epoch, objective), false);
+    return const_cast<CallbackType&>(callback).BeginEpoch(
+        optimizer, function, coordinates, epoch, objective);
   }
 
   template<typename CallbackType,
            typename OptimizerType,
            typename FunctionType,
            typename MatType>
-  static typename std::enable_if<!callbacks::traits::HasBeginEpochSignature<
-      CallbackType, OptimizerType, FunctionType, MatType>::value, bool>::type
+  static typename std::enable_if<callbacks::traits::HasBeginEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid, bool>::type
+  BeginEpochFunction(CallbackType& callback,
+                     OptimizerType& optimizer,
+                     FunctionType& function,
+                     const MatType& coordinates,
+                     const size_t epoch,
+                     const double objective)
+  {
+    const_cast<CallbackType&>(callback).BeginEpoch(
+        optimizer, function, coordinates, epoch, objective);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasBeginEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone, bool>::type
   BeginEpochFunction(CallbackType& /* callback  */,
                      OptimizerType& /* optimizer */,
                      FunctionType& /* function  */,
@@ -769,6 +871,112 @@ class Callback
             function, coordinates)... };
      return result;
   }
+
+ /**
+  * Invoke the GenerationalStepTaken() callback if it exists.
+  * Specialization for MultiObjective case.
+  *
+  * @param callback The callback to call.
+  * @param optimizer The optimizer used to update the function.
+  * @param function Function to optimize.
+  * @param coordinates Starting point.
+  * @param objectives The set of calculated objectives so far.
+  * @param frontIndices The indices of the members belonging to Pareto Front.
+  */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasBool, bool>::type
+  GenerationalStepTakenFunction(CallbackType& callback,
+                                OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                ObjectivesVecType& objectives,
+                                IndicesType& frontIndices)
+  {
+    return const_cast<CallbackType&>(callback).GenerationalStepTaken(
+        optimizer, function, coordinates, objectives, frontIndices);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasVoid, bool>::type
+  GenerationalStepTakenFunction(CallbackType& callback,
+                                OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                ObjectivesVecType& objectives,
+                                IndicesType& frontIndices)
+  {
+    const_cast<CallbackType&>(callback).GenerationalStepTaken(
+        optimizer, function, coordinates, objectives, frontIndices);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  static typename std::enable_if<
+      callbacks::traits::HasGenerationalStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, ObjectivesVecType,
+      IndicesType>::hasNone, bool>::type
+  GenerationalStepTakenFunction(CallbackType& /* callback */,
+                                OptimizerType& /* optimizer */,
+                                FunctionType& /* function */,
+                                MatType& /* coordinates */,
+                                ObjectivesVecType& /* objectives */,
+                                IndicesType& /* frontIndices */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the GenerationalStepTaken() callback if it
+   * exists.
+   *
+   * Specialization for MultiObjective case.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objectives The set of calculated objectives so far.
+   * @param frontIndices The indices of the members belonging to Pareto Front.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename ObjectivesVecType,
+           typename IndicesType,
+           typename MatType,
+           typename ...CallbackTypes>
+  static bool GenerationalStepTaken(OptimizerType& optimizer,
+                                    FunctionType& functions,
+                                    MatType& coordinates,
+                                    ObjectivesVecType& objectives,
+                                    IndicesType& frontIndices,
+                                    CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result = result ||
+        Callback::GenerationalStepTakenFunction(callbacks, optimizer, functions,
+        coordinates, objectives, frontIndices)... };
+    return result;
+  }
 };
 
 } // namespace ens

include/ensmallen_bits/callbacks/early_stop_at_min_loss.hpp

@@ -32,9 +32,9 @@ class EarlyStopAtMinLossType
    * @param patienceIn The number of epochs to wait after the minimum loss has
    *    been reached or no improvement has been made (Default: 10).
    */
-  EarlyStopAtMinLossType<MatType>(const size_t patienceIn = 10) :
-      callbackUsed(false), 
-      patience(patienceIn), 
+  EarlyStopAtMinLossType(const size_t patienceIn = 10) :
+      callbackUsed(false),
+      patience(patienceIn),
       bestObjective(std::numeric_limits<double>::max()),
       steps(0)
   { /* Nothing to do here */ }
@@ -47,13 +47,13 @@ class EarlyStopAtMinLossType
    * @param patienceIn The number of epochs to wait after the minimum loss has
    *    been reached or no improvement has been made (Default: 10).
    */
-  EarlyStopAtMinLossType<MatType>(
+  EarlyStopAtMinLossType(
       std::function<double(const MatType&)> func,
       const size_t patienceIn = 10)
-    : callbackUsed(true), 
-      patience(patienceIn), 
+    : callbackUsed(true),
+      patience(patienceIn),
       bestObjective(std::numeric_limits<double>::max()),
-      steps(0), 
+      steps(0),
       localFunc(func)
   {
     // Nothing to do here
@@ -78,7 +78,7 @@ class EarlyStopAtMinLossType
     if (callbackUsed)
     {
       objective = localFunc(coordinates);
-    } 
+    }
 
     if (objective < bestObjective)
     {
@@ -98,7 +98,8 @@ class EarlyStopAtMinLossType
   }
 
  private:
-  //! False if the first constructor is called, true if the user passed a lambda. 
+  //! False if the first constructor is called, true if the user passed a
+  //! lambda.
   bool callbackUsed;
 
   //! The number of epochs to wait before terminating the optimization process.

include/ensmallen_bits/callbacks/grad_clip_by_norm.hpp

@@ -0,0 +1,61 @@
+/**
+ * @file grad_clip_by_norm.hpp
+ * @author Marcus Edel
+ *
+ * Clip the gradients by multiplying the unit vector of the gradients with the
+ * threshold.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP
+#define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP
+
+namespace ens {
+
+/**
+ * Clip the gradients by multiplying the unit vector of the gradients with the
+ * threshold.
+ */
+class GradClipByNorm
+{
+ public:
+  /**
+   * Set up the gradient clip by norm callback class with the maximum clipping
+   * value.
+   *
+   * @param maxNorm The maximum clipping value.
+   */
+  GradClipByNorm(const double maxNorm) : maxNorm(maxNorm)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                MatType& gradient)
+  {
+    const double gradientNorm = arma::norm(gradient);
+    if (gradientNorm > maxNorm)
+      gradient = maxNorm * gradient / gradientNorm;
+    return false;
+  }
+
+ private:
+  //! The maximum clipping value for gradient clipping.
+  const double maxNorm;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/grad_clip_by_value.hpp

@@ -0,0 +1,61 @@
+/**
+ * @file grad_clip_by_value.hpp
+ * @author Marcus Edel
+ *
+ * Clips the gradient to a specified min and max.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP
+#define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP
+
+namespace ens {
+
+/**
+ * Clip the gradient to a specified min and max.
+ */
+class GradClipByValue
+{
+ public:
+  /**
+   * Set up the gradient clip by value callback class with the min and max
+   * value.
+   *
+   * @param min The minimum value to clip to.
+   * @param max The maximum value to clip to.
+   */
+  GradClipByValue(const double min, const double max) : lower(min), upper(max)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at any call to Gradient().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool Gradient(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                MatType& gradient)
+  {
+    gradient = arma::clamp(gradient, lower, upper);
+    return false;
+  }
+
+ private:
+  //! The minimum value to clip to.
+  const double lower;
+
+  //! The maximum value to clip to.
+  const double upper;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/print_loss.hpp

@@ -38,13 +38,14 @@ class PrintLoss
    * @param objective Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& /* optimizer */,
+  bool EndEpoch(OptimizerType& /* optimizer */,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */,
                 const size_t /* epoch */,
                 const double objective)
   {
     output << objective << std::endl;
+    return false;
   }
 
  private:

include/ensmallen_bits/callbacks/progress_bar.hpp

@@ -98,7 +98,7 @@ class ProgressBar
    * @param objective Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void BeginEpoch(OptimizerType& /* optimizer */,
+  bool BeginEpoch(OptimizerType& /* optimizer */,
                   FunctionType& /* function */,
                   const MatType& /* coordinates */,
                   const size_t epochIn,
@@ -113,6 +113,8 @@ class ProgressBar
 
     epoch = epochIn;
     newEpoch = true;
+
+    return false;
   }
 
   /**
@@ -124,7 +126,7 @@ class ProgressBar
    * @param objective Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void StepTaken(OptimizerType& /* optimizer */,
+  bool StepTaken(OptimizerType& /* optimizer */,
                  FunctionType& /* function */,
                  const MatType& /* coordinates */)
   {
@@ -163,6 +165,8 @@ class ProgressBar
     output.flush();
 
     stepTimer.tic();
+
+    return false;
   }
 
   /**
@@ -174,13 +178,14 @@ class ProgressBar
    * @param objectiveIn Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void Evaluate(OptimizerType& optimizer,
+  bool Evaluate(OptimizerType& optimizer,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */,
                 const double objectiveIn)
   {
     objective += objectiveIn / optimizer.BatchSize();
     steps++;
+    return false;
   }
 
   /**
@@ -193,7 +198,7 @@ class ProgressBar
    * @param objective Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& /* optimizer */,
+  bool EndEpoch(OptimizerType& /* optimizer */,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */,
                 const size_t /* epoch */,
@@ -216,11 +221,12 @@ class ProgressBar
         output << ".";
       }
     }
-
-    const size_t stepTime = epochTimer.toc() / (double) epochSize * 1000;
-    output << "] " << progress << "% - " << (size_t) epochTimer.toc() % 60
-        << "s " << stepTime << "ms/step " << "- loss: " << objective  <<  "\n";
+    const double epochTimerElapsed = epochTimer.toc();
+    const size_t stepTime = epochTimerElapsed / (double) epochSize * 1000;
+    output << "] " << progress << "% - " << epochTimerElapsed
+        << "s/epoch; " << stepTime << "ms/step; loss: " << objective  <<  "\n";
     output.flush();
+    return false;
   }
 
  private:

include/ensmallen_bits/callbacks/query_front.hpp

@@ -0,0 +1,89 @@
+/**
+ * @file query_front.hpp
+ * @author Nanubala Gnana Sai
+ *
+ * Implementation of the query front callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_QUERY_FRONT_HPP
+#define ENSMALLEN_CALLBACKS_QUERY_FRONT_HPP
+
+namespace ens {
+
+/**
+ * Query the current Pareto Front after every GenerationalStepTaken callback function.
+ */
+class QueryFront
+{
+ public:
+  /**
+   * Set up the query front callback class with the specified inputs.
+   *
+   * @param queryRate The frequency at which the Pareto Front is queried.
+   * @param paretoFrontArray A reference to a vector of cube to store the
+   *     queried fronts.
+   */
+  QueryFront(const size_t queryRate,
+             std::vector<arma::cube>& paretoFrontArray) :
+      queryRate(queryRate),
+      paretoFrontArray(paretoFrontArray),
+      genCounter(0)
+  { /* Nothing to do here */ }
+
+  /**
+   * Callback function called at the end of a single generational run.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objectives The set of calculated objectives so far.
+   * @param frontIndices The indices of the members belonging to Pareto Front.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  bool GenerationalStepTaken(OptimizerType& /* opt */,
+                             FunctionType& /* function */,
+                             const MatType& /* coordinates */,
+                             const ObjectivesVecType& objectives,
+                             const IndicesType& frontIndices)
+  {
+    arma::cube currentParetoFront{};
+
+    if (genCounter % queryRate == 0)
+    {
+      currentParetoFront.resize(objectives[0].n_rows, objectives[0].n_cols,
+          frontIndices[0].size());
+      for (size_t solutionIdx = 0; solutionIdx < frontIndices[0].size();
+          ++solutionIdx)
+      {
+        currentParetoFront.slice(solutionIdx) = arma::conv_to<arma::mat>::from(
+            objectives[frontIndices[0][solutionIdx]]);
+      }
+
+      paretoFrontArray.emplace_back(std::move(currentParetoFront));
+    }
+
+    ++genCounter;
+    return false;
+  }
+
+
+ private:
+  //! The rate of query.
+  size_t queryRate;
+  //! A reference to the array of pareto fronts.
+  std::vector<arma::cube>& paretoFrontArray;
+  //! A counter for the current generation.
+  size_t genCounter;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/report.hpp

@@ -87,6 +87,7 @@ class Report
       TruncatePrint(initialCoordinates, outputMatrixSize);
       output << std::endl << "Final coordinates: " << std::endl;
       TruncatePrint(coordinates, outputMatrixSize);
+      output << std::endl;
     }
     else
     {
@@ -239,13 +240,14 @@ class Report
    * @param objective Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void BeginEpoch(OptimizerType& /* optimizer */,
+  bool BeginEpoch(OptimizerType& /* optimizer */,
                   FunctionType& /* function */,
                   const MatType& /* coordinates */,
                   const size_t /* epoch */,
                   const double /* objective */)
   {
     epochCalls++;
+    return false;
   }
 
   /**
@@ -258,7 +260,7 @@ class Report
    * @param objective Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EndEpoch(OptimizerType& optimizer,
+  bool EndEpoch(OptimizerType& optimizer,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */,
                 const size_t /* epoch */,
@@ -282,6 +284,7 @@ class Report
       gradientsNorm.push_back(gradientNorm);
 
     SaveStepSize(optimizer);
+    return false;
   }
 
   /**
@@ -293,7 +296,7 @@ class Report
    * @param objective Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void StepTaken(OptimizerType& optimizer,
+  bool StepTaken(OptimizerType& optimizer,
                  FunctionType& /* function */,
                  const MatType& /* coordinates */)
   {
@@ -307,6 +310,7 @@ class Report
 
       SaveStepSize(optimizer);
     }
+    return false;
   }
 
   /**
@@ -318,13 +322,14 @@ class Report
    * @param objectiveIn Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void Evaluate(OptimizerType& /* optimizer */,
+  bool Evaluate(OptimizerType& /* optimizer */,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */,
                 const double objectiveIn)
   {
     objective = objectiveIn;
     evaluateCalls++;
+    return false;
   }
 
   /**
@@ -337,7 +342,7 @@ class Report
     * @param objectiveIn Objective value of the current point.
     */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void EvaluateConstraint(OptimizerType& /* optimizer */,
+  bool EvaluateConstraint(OptimizerType& /* optimizer */,
                           FunctionType& /* function */,
                           const MatType& /* coordinates */,
                           const size_t /* constraint */,
@@ -345,6 +350,7 @@ class Report
   {
     objective += objectiveIn;
     evaluateCalls++;
+    return false;
   }
 
   /**
@@ -356,7 +362,7 @@ class Report
    * @param gradientIn Matrix that holds the gradient.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void Gradient(OptimizerType& /* optimizer */,
+  bool Gradient(OptimizerType& /* optimizer */,
                 FunctionType& /* function */,
                 const MatType& /* coordinates */,
                 const MatType& gradientIn)
@@ -364,6 +370,7 @@ class Report
     hasGradient = true;
     gradientNorm = arma::norm(gradientIn);
     gradientCalls++;
+    return false;
   }
 
   /**
@@ -376,13 +383,14 @@ class Report
    * @param gradient Matrix that holds the gradient;
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void GradientConstraint(OptimizerType& optimizer,
+  bool GradientConstraint(OptimizerType& optimizer,
                           FunctionType& function,
                           const MatType& coordinates,
                           const size_t /* constraint */,
                           const MatType& gradient)
   {
     Gradient(optimizer, function, coordinates, gradient);
+    return false;
   }
 
  private:

include/ensmallen_bits/callbacks/store_best_coordinates.hpp

@@ -40,7 +40,7 @@ class StoreBestCoordinates
    * @param objective Objective value of the current point.
    */
   template<typename OptimizerType, typename FunctionType, typename MatType>
-  void Evaluate(OptimizerType& /* optimizer */,
+  bool Evaluate(OptimizerType& /* optimizer */,
                 FunctionType& /* function */,
                 const MatType& coordinates,
                 const double objective)
@@ -50,6 +50,7 @@ class StoreBestCoordinates
       bestObjective = objective;
       bestCoordinates = coordinates;
     }
+    return false;
   }
 
   //! Get the best coordinates.

include/ensmallen_bits/callbacks/timer_stop.hpp

@@ -45,6 +45,30 @@ class TimerStop
     timer.tic();
   }
 
+  /**
+   * Callback function called when a step is taken.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */)
+  {
+    if (timer.toc() > duration)
+    {
+      Info << "Timer timeout (" << duration << "s) reached; terminating "
+          << "optimization." << std::endl;
+      return true;
+    }
+
+    return false;
+  }
+
   /**
    * Callback function called at the end of a pass over the data.
    *
@@ -63,7 +87,8 @@ class TimerStop
   {
     if (timer.toc() > duration)
     {
-      Info << "Timer timeout reached; terminate optimization." << std::endl;
+      Info << "Timer timeout (" << duration << "s) reached; terminating "
+          << "optimization." << std::endl;
       return true;
     }
 

include/ensmallen_bits/callbacks/traits.hpp

@@ -37,6 +37,8 @@ ENS_HAS_EXACT_METHOD_FORM(BeginEpoch, HasBeginEpoch)
 ENS_HAS_EXACT_METHOD_FORM(EndEpoch, HasEndEpoch)
 //! Detect an StepTaken() method.
 ENS_HAS_EXACT_METHOD_FORM(StepTaken, HasStepTaken)
+//! Detect an GenerationalStepTaken() method.
+ENS_HAS_EXACT_METHOD_FORM(GenerationalStepTaken, HasGenerationalStepTaken)
 
 template<typename OptimizerType,
          typename FunctionType,
@@ -47,7 +49,7 @@ struct TypedForms
   //! This is the form of a bool Evaluate() callback method.
   template<typename CallbackType>
   using EvaluateBoolForm =
-      void(CallbackType::*)(OptimizerType&,
+      bool(CallbackType::*)(OptimizerType&,
                             FunctionType&,
                             const MatType&,
                             const double);
@@ -63,7 +65,7 @@ struct TypedForms
   //! This is the form of a bool EvaluateConstraint() callback method.
   template<typename CallbackType>
   using EvaluateConstraintBoolForm =
-      void(CallbackType::*)(OptimizerType&,
+      bool(CallbackType::*)(OptimizerType&,
                             FunctionType&,
                             const MatType&,
                             const size_t,
@@ -81,11 +83,20 @@ struct TypedForms
   //! This is the form of a bool Gradient() callback method.
   template<typename CallbackType>
   using GradientBoolForm =
-      void(CallbackType::*)(OptimizerType&,
+      bool(CallbackType::*)(OptimizerType&,
                             FunctionType&,
                             const MatType&,
                             const MatType&);
 
+  //! This is the form of a bool Gradient() callback method where the gradient
+  //! is modifiable.
+  template<typename CallbackType>
+  using GradientBoolModifiableForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            MatType&);
+
   //! This is the form of a void Gradient() callback method.
   template<typename CallbackType>
   using GradientVoidForm =
@@ -94,15 +105,34 @@ struct TypedForms
                             const MatType&,
                             const MatType&);
 
+  //! This is the form of a void Gradient() callback method where the gradient
+  //! is modifiable.
+  template<typename CallbackType>
+  using GradientVoidModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            MatType&);
+
   //! This is the form of a bool GradientConstraint() callback method.
   template<typename CallbackType>
   using GradientConstraintBoolForm =
-      void(CallbackType::*)(OptimizerType&,
+      bool(CallbackType::*)(OptimizerType&,
                             FunctionType&,
                             const MatType&,
                             const size_t,
                             const MatType&);
 
+  //! This is the form of a bool GradientConstraint() callback method where the
+  //! gradient is modifiable.
+  template<typename CallbackType>
+  using GradientConstraintBoolModifiableForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            MatType&);
+
   //! This is the form of a void GradientConstraint() callback method.
   template<typename CallbackType>
   using GradientConstraintVoidForm =
@@ -112,6 +142,16 @@ struct TypedForms
                             const size_t,
                             const MatType&);
 
+  //! This is the form of a void GradientConstraint() callback method where the
+  //! gradient is modifiable.
+  template<typename CallbackType>
+  using GradientConstraintVoidModifiableForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            MatType&);
+
   //! This is the form of a bool BeginOptimization() callback method.
   template<typename CallbackType>
   using BeginOptimizationBoolForm =
@@ -199,23 +239,11 @@ template<typename CallbackType,
          typename MatType>
 struct HasBeginOptimizationSignature
 {
-  const static bool hasBool =
+  constexpr static bool value =
+      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationBoolForm>::value ||
       HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
-      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
-
-  const static bool hasVoid =
-      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
-      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-         FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
-
-  const static bool hasNone =
-      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
-      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
-         FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
 };
 
 //! Utility struct, check if either void Evaluate() or bool Evaluate()
@@ -226,11 +254,23 @@ template<typename CallbackType,
          typename MatType>
 struct HasEvaluateSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
       HasEvaluate<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateBoolForm>::value ||
+          FunctionType, MatType>::template EvaluateBoolForm>::value &&
+      !HasEvaluate<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateVoidForm>::value;
+
+  constexpr static bool hasVoid =
+      !HasEvaluate<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateBoolForm>::value &&
       HasEvaluate<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateVoidForm>::value;
+          FunctionType, MatType>::template EvaluateVoidForm>::value;
+
+  constexpr static bool hasNone =
+      !HasEvaluate<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateBoolForm>::value &&
+      !HasEvaluate<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateVoidForm>::value;
 };
 
 //! Utility struct, check if either void EvaluateConstraint() or
@@ -241,11 +281,23 @@ template<typename CallbackType,
          typename MatType>
 struct HasEvaluateConstraintSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
       HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateConstraintBoolForm>::value ||
+          FunctionType, MatType>::template EvaluateConstraintBoolForm>::value &&
+      !HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+
+  constexpr static bool hasVoid =
+      !HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateConstraintBoolForm>::value &&
       HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+          FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+
+  constexpr static bool hasNone =
+      !HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateConstraintBoolForm>::value &&
+      !HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
 };
 
 //! Utility struct, check if either void Gradient() or bool Gradient()
@@ -257,11 +309,41 @@ template<typename CallbackType,
          typename Gradient>
 struct HasGradientSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
+      (HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
       HasGradient<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
+          FunctionType, MatType, Gradient>::template
+          GradientBoolModifiableForm>::value) &&
+      (!HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientVoidForm>::value ||
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientVoidModifiableForm>::value);
+
+  constexpr static bool hasVoid =
+      (!HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientBoolModifiableForm>::value) &&
+      (HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientVoidForm>::value ||
       HasGradient<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType, Gradient>::template GradientVoidForm>::value;
+          FunctionType, MatType, Gradient>::template
+          GradientVoidModifiableForm>::value);
+
+  constexpr static bool hasNone =
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientBoolForm>::value &&
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientBoolModifiableForm>::value &&
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template GradientVoidForm>::value &&
+      !HasGradient<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientVoidModifiableForm>::value;
 };
 
 //! Utility struct, check if either void GradientConstraint() or
@@ -273,13 +355,29 @@ template<typename CallbackType,
          typename Gradient>
 struct HasGradientConstraintSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
       HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType,
-          Gradient>::template GradientConstraintBoolForm>::value ||
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintBoolForm>::value &&
+      !HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintVoidForm>::value;
+
+  constexpr static bool hasVoid =
+      !HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintBoolForm>::value &&
       HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType,
-          Gradient>::template GradientConstraintVoidForm>::value;
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintVoidForm>::value;
+
+  constexpr static bool hasNone =
+      !HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintBoolForm>::value &&
+      !HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType, Gradient>::template
+          GradientConstraintVoidForm>::value;
 };
 
 //! Utility struct, check if either void EndOptimization() or
@@ -290,11 +388,11 @@ template<typename CallbackType,
          typename MatType>
 struct HasEndOptimizationSignature
 {
-  const static bool value =
+  constexpr static bool value =
       HasEndOptimization<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EndOptimizationBoolForm>::value ||
+          FunctionType, MatType>::template EndOptimizationBoolForm>::value ||
       HasEndOptimization<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template EndOptimizationVoidForm>::value;
+          FunctionType, MatType>::template EndOptimizationVoidForm>::value;
 };
 
 //! Utility struct, check if either void BeginEpoch() or bool BeginEpoch()
@@ -305,11 +403,23 @@ template<typename CallbackType,
          typename MatType>
 struct HasBeginEpochSignature
 {
-  const static bool value =
+  constexpr static bool hasBool =
       HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template BeginEpochBoolForm>::value ||
+          FunctionType, MatType>::template BeginEpochBoolForm>::value &&
+      !HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginEpochVoidForm>::value;
+
+  constexpr static bool hasVoid =
+      !HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginEpochBoolForm>::value &&
       HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
-      FunctionType, MatType>::template BeginEpochVoidForm>::value;
+          FunctionType, MatType>::template BeginEpochVoidForm>::value;
+
+  constexpr static bool hasNone =
+      !HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginEpochBoolForm>::value &&
+      !HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginEpochVoidForm>::value;
 };
 
 //! Utility struct, check if either void EndEpoch() or bool EndEpoch()
@@ -320,19 +430,19 @@ template<typename CallbackType,
          typename MatType>
 struct HasEndEpochSignature
 {
-  const static bool hasBool =
+  constexpr static bool hasBool =
       HasEndEpoch<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template EndEpochBoolForm>::value &&
       !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template EndEpochVoidForm>::value;
 
-  const static bool hasVoid =
+  constexpr static bool hasVoid =
       !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template EndEpochBoolForm>::value &&
       HasEndEpoch<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template EndEpochVoidForm>::value;
 
-  const static bool hasNone =
+  constexpr static bool hasNone =
       !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template EndEpochBoolForm>::value &&
       !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
@@ -346,25 +456,88 @@ template<typename CallbackType,
          typename MatType>
 struct HasStepTakenSignature
 {
-  const static bool hasBool =
+  constexpr static bool hasBool =
       HasStepTaken<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template StepTakenBoolForm>::value &&
       !HasStepTaken<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template StepTakenVoidForm>::value;
 
-  const static bool hasVoid =
+  constexpr static bool hasVoid =
       !HasStepTaken<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template StepTakenBoolForm>::value &&
       HasStepTaken<CallbackType, TypedForms<OptimizerType,
-         FunctionType, MatType>::template StepTakenVoidForm>::value;
+          FunctionType, MatType>::template StepTakenVoidForm>::value;
 
-  const static bool hasNone =
+  constexpr static bool hasNone =
       !HasStepTaken<CallbackType, TypedForms<OptimizerType,
           FunctionType, MatType>::template StepTakenBoolForm>::value &&
       !HasStepTaken<CallbackType, TypedForms<OptimizerType,
-         FunctionType, MatType>::template StepTakenVoidForm>::value;
+          FunctionType, MatType>::template StepTakenVoidForm>::value;
 };
 
+//! A utility struct for Typed Forms required in
+//! callbacks for MultiObjective Optimizers.
+template<typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename GradType = MatType>
+struct MOOTypedForms
+{
+  //! This is the form of a bool GenerationalStepTaken() for MOO callback method.
+  template<typename CallbackType>
+  using GenerationalStepTakenBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const ObjectivesVecType&,
+                            const IndicesType&);
+
+  //! This is the form of a void StepTaken() for MOO callback method.
+  template<typename CallbackType>
+  using GenerationalStepTakenVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const ObjectivesVecType&,
+                            const IndicesType&);
+};
+
+//! Utility struct, check if either void StepTaken() or bool StepTaken() exists.
+//! Specialization for Multiobjective case.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename ObjectivesVecType,
+         typename IndicesType,
+         typename MatType>
+ struct HasGenerationalStepTakenSignature
+{
+  constexpr static bool hasBool =
+    HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+
+  constexpr static bool hasVoid =
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+
+  constexpr static bool hasNone =
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenBoolForm>::value &&
+    !HasGenerationalStepTaken<CallbackType, MOOTypedForms<OptimizerType,
+        FunctionType, MatType, ObjectivesVecType, IndicesType>::
+        template GenerationalStepTakenVoidForm>::value;
+};
 } // namespace traits
 } // namespace callbacks
 } // namespace ens

include/ensmallen_bits/cd/cd.hpp

@@ -1,16 +1,16 @@
 /**
- * @file scd.hpp
+ * @file cd.hpp
  * @author Shikhar Bhardwaj
  *
- * Stochastic Coordinate Descent (SCD).
+ * Coordinate Descent (CD).
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
  * the terms of the 3-clause BSD license.  You should have received a copy of
  * the 3-clause BSD license along with ensmallen.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef ENSMALLEN_SCD_SCD_HPP
-#define ENSMALLEN_SCD_SCD_HPP
+#ifndef ENSMALLEN_CD_CD_HPP
+#define ENSMALLEN_CD_CD_HPP
 
 #include "descent_policies/cyclic_descent.hpp"
 #include "descent_policies/random_descent.hpp"
@@ -42,7 +42,7 @@ namespace ens {
  * }
  * @endcode
  *
- * SCD can optimize partially differentiable functions.  For more details, see
+ * CD can optimize partially differentiable functions.  For more details, see
  * the documentation on function types included with this distribution or on the
  * ensmallen website.
  *
@@ -50,11 +50,11 @@ namespace ens {
  *     coordinate for descent is selected.
  */
 template <typename DescentPolicyType = RandomDescent>
-class SCD
+class CD
 {
  public:
   /**
-   * Construct the SCD optimizer with the given function and parameters. The
+   * Construct the CD optimizer with the given function and parameters. The
    * default value here are not necessarily good for every problem, so it is
    * suggested that the values used are tailored for the task at hand. The
    * maximum number of iterations refers to the maximum number of "descents"
@@ -70,11 +70,11 @@ class SCD
    * @param descentPolicy The policy to use for picking up the coordinate to
    *    descend on.
    */
-  SCD(const double stepSize = 0.01,
-      const size_t maxIterations = 100000,
-      const double tolerance = 1e-5,
-      const size_t updateInterval = 1e3,
-      const DescentPolicyType descentPolicy = DescentPolicyType());
+  CD(const double stepSize = 0.01,
+     const size_t maxIterations = 100000,
+     const double tolerance = 1e-5,
+     const size_t updateInterval = 1e3,
+     const DescentPolicyType descentPolicy = DescentPolicyType());
 
   /**
    * Optimize the given function using stochastic coordinate descent. The
@@ -94,7 +94,7 @@ class SCD
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(ResolvableFunctionType& function,
            MatType& iterate,
@@ -158,6 +158,24 @@ class SCD
 } // namespace ens
 
 // Include implementation.
-#include "scd_impl.hpp"
+#include "cd_impl.hpp"
+
+namespace ens {
+
+/**
+ * Backwards-compatibility alias; this can be removed after ensmallen 3.10.0.
+ * The history here is that CD was originally named SCD, but that is an
+ * inaccurate name because this is not a stochastic technique; thus, it was
+ * renamed SCD.
+ */
+template<typename DescentPolicyType = RandomDescent>
+using SCD = CD<DescentPolicyType>;
+
+// Convenience typedefs.
+using RandomCD = CD<RandomDescent>;
+using GreedyCD = CD<GreedyDescent>;
+using CyclicCD = CD<CyclicDescent>;
+
+} // namespace ens
 
 #endif

include/ensmallen_bits/cd/cd_impl.hpp

@@ -1,26 +1,26 @@
 /**
- * @file scd_impl.hpp
+ * @file cd_impl.hpp
  * @author Shikhar Bhardwaj
  *
- * Implementation of stochastic coordinate descent.
+ * Implementation of coordinate descent.
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
  * the terms of the 3-clause BSD license.  You should have received a copy of
  * the 3-clause BSD license along with ensmallen.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef ENSMALLEN_SCD_SCD_IMPL_HPP
-#define ENSMALLEN_SCD_SCD_IMPL_HPP
+#ifndef ENSMALLEN_CD_CD_IMPL_HPP
+#define ENSMALLEN_CD_CD_IMPL_HPP
 
 // In case it hasn't been included yet.
-#include "scd.hpp"
+#include "cd.hpp"
 
 #include <ensmallen_bits/function.hpp>
 
 namespace ens {
 
 template <typename DescentPolicyType>
-SCD<DescentPolicyType>::SCD(
+CD<DescentPolicyType>::CD(
     const double stepSize,
     const size_t maxIterations,
     const double tolerance,
@@ -39,9 +39,9 @@ template <typename ResolvableFunctionType,
           typename MatType,
           typename GradType,
           typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
-SCD<DescentPolicyType>::Optimize(
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
+CD<DescentPolicyType>::Optimize(
     ResolvableFunctionType& function,
     MatType& iterateIn,
     CallbackTypes&&... callbacks)
@@ -66,10 +66,12 @@ SCD<DescentPolicyType>::Optimize(
   // Controls early termination of the optimization process.
   bool terminate = false;
 
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
   // Start iterating.
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
-  for (size_t i = 1; i != maxIterations && !terminate; ++i)
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     // Get the coordinate to descend on.
     size_t featureIdx = descentPolicy.template DescentFeature<
@@ -81,9 +83,11 @@ SCD<DescentPolicyType>::Optimize(
 
     terminate |= Callback::Gradient(*this, function, iterate, overallObjective,
         gradient, callbacks...);
+    if (terminate)
+      break;
 
     // Update the decision variable with the partial gradient.
-    iterate.col(featureIdx) -= stepSize * gradient.col(featureIdx);
+    iterate.col(featureIdx) -= ElemType(stepSize) * gradient.col(featureIdx);
     terminate |= Callback::StepTaken(*this, function, iterate, callbacks...);
 
     // Check for convergence.
@@ -94,12 +98,12 @@ SCD<DescentPolicyType>::Optimize(
           overallObjective, callbacks...);
 
       // Output current objective function.
-      Info << "SCD: iteration " << i << ", objective " << overallObjective
+      Info << "CD: iteration " << i << ", objective " << overallObjective
           << "." << std::endl;
 
       if (std::isnan(overallObjective) || std::isinf(overallObjective))
       {
-        Warn << "SCD: converged to " << overallObjective << "; terminating"
+        Warn << "CD: converged to " << overallObjective << "; terminating"
             << " with failure.  Try a smaller step size?" << std::endl;
 
         Callback::EndOptimization(*this, function, iterate, callbacks...);
@@ -108,7 +112,7 @@ SCD<DescentPolicyType>::Optimize(
 
       if (std::abs(lastObjective - overallObjective) < tolerance)
       {
-        Info << "SCD: minimized within tolerance " << tolerance << "; "
+        Info << "CD: minimized within tolerance " << tolerance << "; "
             << "terminating optimization." << std::endl;
 
         Callback::EndOptimization(*this, function, iterate, callbacks...);
@@ -119,12 +123,16 @@ SCD<DescentPolicyType>::Optimize(
     }
   }
 
-  Info << "SCD: maximum iterations (" << maxIterations << ") reached; "
-      << "terminating optimization." << std::endl;
-
-  // Calculate and return final objective.
+  if (!terminate)
+  {
+    Info << "CD: maximum iterations (" << maxIterations << ") reached; "
+        << "terminating optimization." << std::endl;
+  }
+  
+  // Calculate and return final objective.  No need to pay attention to the
+  // result of the callback.
   const ElemType objective = function.Evaluate(iterate);
-  Callback::Evaluate(*this, function, iterate, objective, callbacks...);
+  (void) Callback::Evaluate(*this, function, iterate, objective, callbacks...);
 
   Callback::EndOptimization(*this, function, iterate, callbacks...);
   return objective;

include/ensmallen_bits/cd/descent_policies/random_descent.hpp

@@ -52,8 +52,11 @@ class RandomDescent
                                const MatType& /* iterate */,
                                const ResolvableFunctionType& function)
   {
+    // return randi<size_t>(
+    //     arma::distr_param(0, function.NumFeatures() - 1));
+
     return arma::as_scalar(arma::randi<arma::uvec>(
-          1, arma::distr_param(0, function.NumFeatures() - 1)));
+      1, arma::distr_param(0, function.NumFeatures() - 1)));
   }
 };
 

include/ensmallen_bits/cmaes/active_cmaes.hpp

@@ -0,0 +1,190 @@
+/**
+ * @file active_cmaes.hpp
+ * @author Marcus Edel
+ * @author Suvarsha Chennareddy
+ *
+ * Definition of the Active Covariance Matrix Adaptation Evolution Strategy
+ * as proposed by G.A Jastrebski and D.V Arnold in "Improving Evolution
+ * Strategies through Active Covariance Matrix Adaptation".
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_ACTIVE_CMAES_HPP
+#define ENSMALLEN_CMAES_ACTIVE_CMAES_HPP
+
+#include "full_selection.hpp"
+#include "random_selection.hpp"
+#include "transformation_policies/empty_transformation.hpp"
+#include "transformation_policies/boundary_box_constraint.hpp"
+
+namespace ens {
+
+/**
+ * Active CMA-ES is a variant of the stochastic search algorithm
+ * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
+ * Active CMA-ES actively reduces the uncertainty in unfavourable directions by
+ * exploiting the information about bad mutations in the covariance matrix
+ * update step. This isn't for the purpose of accelerating progress, but
+ * instead for speeding up the adaptation of the covariance matrix (which, in
+ * turn, will lead to faster progress).
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @INPROCEEDINGS{1688662,
+ *   author    = {Jastrebski, G.A. and Arnold, D.V.},
+ *   booktitle = {2006 IEEE International Conference on Evolutionary
+ *                Computation},
+ *   title     = {Improving Evolution Strategies through Active Covariance
+ *                Matrix Adaptation},
+ *   year      = {2006},
+ *   volume    = {},
+ *   number    = {},
+ *   pages     = {2814-2821},
+ *   doi       = {10.1109/CEC.2006.1688662}}
+ * @endcode
+ *
+ * Active CMA-ES can optimize separable functions.  For more details, see the
+ * documentation on function types included with this distribution or on the
+ * ensmallen website.
+ *
+ * @tparam SelectionPolicy The selection strategy used for the evaluation step.
+ * @tparam TransformationPolicy The transformation strategy used to
+ *       map decision variables to the desired domain during fitness evaluation
+ *       and termination. Use EmptyTransformation if the domain isn't bounded.
+ */
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
+class ActiveCMAES
+{
+ public:
+  /**
+   * Construct the Active CMA-ES optimizer with the given function and
+   * parameters. The defaults here are not necessarily good for the given
+   * problem, so it is suggested that the values used be tailored to the task at
+   * hand.  The maximum number of iterations refers to the maximum number of
+   * points that are processed (i.e., one iteration equals one point; one
+   * iteration does not equal one pass over the dataset).
+   *
+   * @param lambda The population size (0 use the default size).
+   * @param transformationPolicy Instantiated transformation policy used to
+   *     map the coordinates to the desired domain.
+   * @param batchSize Batch size to use for the objective calculation.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param selectionPolicy Instantiated selection policy used to calculate the
+   *     objective.
+   * @param stepSize Starting sigma/step size (will be modified).
+   */
+  ActiveCMAES(
+      const size_t lambda = 0,
+      const TransformationPolicyType&
+          transformationPolicy = TransformationPolicyType(),
+      const size_t batchSize = 32,
+      const size_t maxIterations = 1000,
+      const double tolerance = 1e-5,
+      const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+      double stepSize = 0);
+
+  /**
+   * Optimize the given function using Active CMA-ES. The given starting point
+   * will be modified to store the finishing point of the algorithm, and the
+   * final objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+      typename MatType,
+      typename... CallbackTypes>
+      typename MatType::elem_type Optimize(
+          SeparableFunctionType& function,
+          MatType& iterate,
+          CallbackTypes&&... callbacks);
+
+  //! Get the population size.
+  size_t PopulationSize() const { return lambda; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return lambda; }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return batchSize; }
+  //! Modify the batch size.
+  size_t& BatchSize() { return batchSize; }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return maxIterations; }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return maxIterations; }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return tolerance; }
+
+  //! Get the selection policy.
+  const SelectionPolicyType& SelectionPolicy() const { return selectionPolicy; }
+  //! Modify the selection policy.
+  SelectionPolicyType& SelectionPolicy() { return selectionPolicy; }
+
+  //! Get the transformation policy.
+  const TransformationPolicyType& TransformationPolicy() const
+  { return transformationPolicy; }
+  //! Modify the transformation policy.
+  TransformationPolicyType& TransformationPolicy()
+  { return transformationPolicy; }
+
+  //! Get the step size.
+  double StepSize() const
+  { return stepSize; }
+  //! Modify the step size.
+  double& StepSize()
+  { return stepSize; }
+
+ private:
+  //! Population size.
+  size_t lambda;
+
+  //! The batch size for processing.
+  size_t batchSize;
+
+  //! The maximum number of allowed iterations.
+  size_t maxIterations;
+
+  //! The tolerance for termination.
+  double tolerance;
+
+  //! The selection policy used to calculate the objective.
+  SelectionPolicyType selectionPolicy;
+
+  //! The transformationPolicy used to map coordinates to the suitable domain
+  //! while evaluating fitness. This mapping is also done after optimization
+  //! has completed.
+  TransformationPolicyType transformationPolicy;
+
+  //! The step size.
+  double stepSize;
+};
+
+/**
+ * Convenient typedef for Active CMAES approximation.
+ */
+template<typename TransformationPolicyType = EmptyTransformation<>,
+         typename SelectionPolicyType = RandomSelection>
+using ApproxActiveCMAES = ActiveCMAES<SelectionPolicyType, TransformationPolicyType>;
+
+} // namespace ens
+
+// Include implementation.
+#include "active_cmaes_impl.hpp"
+
+#endif

include/ensmallen_bits/cmaes/active_cmaes_impl.hpp

@@ -0,0 +1,354 @@
+/**
+ * @file active_cmaes_impl.hpp
+ * @author Marcus Edel
+ * @author Suvarsha Chennareddy
+ *
+ * Implementation of the Active Covariance Matrix Adaptation Evolution Strategy
+ * as proposed by G.A Jastrebski and D.V Arnold in "Improving Evolution
+ * Strategies through Active Covariance Matrix Adaptation".
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_ACTIVE_CMAES_IMPL_HPP
+#define ENSMALLEN_CMAES_ACTIVE_CMAES_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "active_cmaes.hpp"
+
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+template<typename SelectionPolicyType, typename TransformationPolicyType>
+ActiveCMAES<SelectionPolicyType, TransformationPolicyType>::ActiveCMAES(
+                                  const size_t lambda,
+                                  const TransformationPolicyType& 
+                                        transformationPolicy,
+                                  const size_t batchSize,
+                                  const size_t maxIterations,
+                                  const double tolerance,
+                                  const SelectionPolicyType& selectionPolicy,
+                                  double stepSizeIn) :
+    lambda(lambda),
+    batchSize(batchSize),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    selectionPolicy(selectionPolicy),
+    transformationPolicy(transformationPolicy),
+    stepSize(stepSizeIn)
+{ /* Nothing to do. */ }
+
+//! Optimize the function (minimize).
+template<typename SelectionPolicyType, typename TransformationPolicyType>
+template<typename SeparableFunctionType,
+         typename MatType,
+         typename... CallbackTypes>
+typename MatType::elem_type ActiveCMAES<SelectionPolicyType, 
+  TransformationPolicyType>::Optimize(
+    SeparableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  typedef typename ForwardType<MatType>::bcol BaseColType;
+  typedef typename ForwardType<MatType>::uvec UVecType;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitrarySeparableFunctionTypeAPI<
+      SeparableFunctionType, BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Find the number of functions to use.
+  const size_t numFunctions = function.NumFunctions();
+
+  // Population size.
+  if (lambda == 0)
+    lambda = (4 + std::round(3 * std::log(iterate.n_elem))) * 10;
+
+  // Parent number.
+  const size_t mu = std::round(lambda / 4);
+
+  // Recombination weight (w = 1 / (parent number)).
+  const ElemType w = 1.0 / mu;
+
+  // Number of effective solutions.
+  const ElemType muEffective = mu;
+
+  // Step size control parameters.
+  BaseMatType sigma(2, 1); // sigma is vector-shaped.
+  if (stepSize == 0)
+    sigma(0) = transformationPolicy.InitialStepSize();
+  else
+    sigma(0) = ElemType(stepSize);
+
+  const ElemType cs = 4 / ElemType(iterate.n_elem + 4);
+  const ElemType ds = 1 + cs;
+  const ElemType enn = std::sqrt(iterate.n_elem) * (1 -
+      1 / ElemType(4 * iterate.n_elem) +
+      1 / ElemType(21 * std::pow(iterate.n_elem, 2)));
+
+  // Covariance update parameters. Cumulation for distribution.
+  const ElemType cc = cs;
+  const ElemType ccov = 2 /
+      std::pow((iterate.n_elem + std::sqrt(ElemType(2))), ElemType(2));
+  const ElemType beta = (4 * mu - 2) /
+      (std::pow(ElemType(iterate.n_elem + 12), ElemType(2)) + 4 * mu);
+
+  std::vector<BaseMatType> mPosition(2, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  mPosition[0] = iterate;
+
+  BaseMatType step(iterate.n_rows, iterate.n_cols);
+  step.zeros();
+
+  BaseMatType transformedIterate = transformationPolicy.Transform(iterate);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // Calculate the first objective function.
+  ElemType currentObjective = 0;
+  for (size_t f = 0; f < numFunctions; f += batchSize)
+  {
+    const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
+    const ElemType objective = function.Evaluate(transformedIterate, f,
+        effectiveBatchSize);
+    currentObjective += objective;
+
+    terminate |= Callback::Evaluate(*this, function, transformedIterate,
+        objective, callbacks...);
+  }
+
+  ElemType overallObjective = currentObjective;
+  ElemType lastObjective = std::numeric_limits<ElemType>::max();
+
+  // Population parameters.
+  std::vector<BaseMatType> pStep(lambda, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  std::vector<BaseMatType> pPosition(lambda, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  BaseMatType pObjective(lambda, 1); // pObjective is vector-shaped.
+  std::vector<BaseMatType> ps(2, BaseMatType(iterate.n_rows, iterate.n_cols));
+  ps[0].zeros();
+  ps[1].zeros();
+  std::vector<BaseMatType> pc = ps;
+  std::vector<BaseMatType> C(2, BaseMatType(iterate.n_elem, iterate.n_elem));
+  C[0].eye();
+
+  // Covariance matrix parameters.
+  BaseColType eigval;
+  BaseMatType eigvec;
+  BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped.
+  eigvalZero.zeros();
+
+  // The current visitation order (sorted by population objectives).
+  UVecType idx = linspace<UVecType>(0, lambda - 1, lambda);
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
+  // Now iterate!
+  Callback::BeginOptimization(*this, function, transformedIterate,
+      callbacks...);
+
+  size_t idx0, idx1;
+
+  // The number of generations to wait after the minimum loss has
+  // been reached or no improvement has been made before terminating.
+  size_t patience = 10 + (30 * iterate.n_elem / lambda) + 1;
+  size_t steps = 0;
+
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
+  {
+    // To keep track of where we are.
+    idx0 = i % 2;
+    idx1 = (i + 1) % 2;
+
+    // Perform Cholesky decomposition. If the matrix is not positive definite,
+    // add a small value and try again.
+    BaseMatType covLower;
+    // while (!arma::chol(covLower, C[idx0], "lower"))
+    while (!chol(covLower, C[idx0]))
+      C[idx0].diag() += std::numeric_limits<ElemType>::epsilon();
+
+    eig_sym(eigval, eigvec, C[idx0]);
+
+    for (size_t j = 0; j < lambda; ++j)
+    {
+      if (iterate.n_rows > iterate.n_cols)
+      {
+        pStep[idx(j)] = covLower *
+          randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
+      }
+      else
+      {
+        pStep[idx(j)] = randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
+          * covLower.t();
+      }
+
+      pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];
+
+      // Calculate the objective function.
+      pObjective(idx(j)) = selectionPolicy.Select(function, batchSize,
+          transformationPolicy.Transform(pPosition[idx(j)]), terminate,
+          callbacks...);
+    }
+
+    // Sort population.
+    idx = sort_index(pObjective);
+
+    step = w * pStep[idx(0)];
+    for (size_t j = 1; j < mu; ++j)
+      step += w * pStep[idx(j)];
+
+    mPosition[idx1] = mPosition[idx0] + sigma(idx0) * step;
+
+    // Calculate the objective function.
+    currentObjective = selectionPolicy.Select(function, batchSize,
+        transformationPolicy.Transform(mPosition[idx1]), terminate,
+        callbacks...);
+
+    // Update best parameters.
+    if (currentObjective < overallObjective)
+    {
+      overallObjective = currentObjective;
+      iterate = mPosition[idx1];
+
+      transformedIterate = transformationPolicy.Transform(iterate);
+      terminate |= Callback::StepTaken(*this, function,
+          transformedIterate, callbacks...);
+    }
+
+    // Update Step Size.
+    if (iterate.n_rows > iterate.n_cols)
+    {
+      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
+        cs * (2 - cs) * muEffective) *
+        eigvec * diagmat(1 / eigval) * eigvec.t() * step;
+    }
+    else
+    {
+      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
+          cs * (2 - cs) * muEffective) * step *
+          eigvec * diagmat(1 / eigval) * eigvec.t();
+    }
+
+    const ElemType psNorm = norm(ps[idx1]);
+    sigma(idx1) = sigma(idx0) * std::exp(cs / ds * (psNorm / enn - 1));
+
+    if (std::isnan(sigma(idx1)) || sigma(idx1) > 1e14)
+    {
+      Warn << "The step size diverged to " << sigma(idx1) << "; "
+          << "terminating with failure.  Try a smaller step size?" << std::endl;
+
+      iterate = transformationPolicy.Transform(iterate);
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
+      return overallObjective;
+    }
+
+    pc[idx1] = (1 - cc) * pc[idx0] + std::sqrt(cc * (2 - cc) *
+        muEffective) * step;
+
+    if (iterate.n_rows > iterate.n_cols)
+    {
+      C[idx1] = (1 - ccov) * C[idx0] + ccov *
+          (pc[idx1] * pc[idx1].t());
+
+      for (size_t j = 0; j < mu; ++j)
+      {
+        C[idx1] = C[idx1] + beta * w *
+            pStep[idx(j)] * pStep[idx(j)].t();
+      }
+
+      for (size_t j = lambda - mu; j < lambda; ++j)
+      {
+        C[idx1] = C[idx1] - beta * w *
+            pStep[idx(j)] * pStep[idx(j)].t();
+      }
+    }
+    else
+    {
+      C[idx1] = (1 - ccov) * C[idx0] + ccov *
+          (pc[idx1].t() * pc[idx1]);
+
+      for (size_t j = 0; j < mu; ++j)
+      {
+        C[idx1] = C[idx1] + beta * w *
+            pStep[idx(j)].t() * pStep[idx(j)];
+      }
+
+      for (size_t j = lambda - mu; j < lambda; ++j)
+      {
+        C[idx1] = C[idx1] - beta * w *
+            pStep[idx(j)].t() * pStep[idx(j)];
+      }
+    }
+
+    eig_sym(eigval, eigvec, C[idx1]);
+    const UVecType negativeEigval = find(eigval < 0, 1);
+    if (!negativeEigval.is_empty())
+    {
+      if (negativeEigval(0) == 0)
+      {
+        C[idx1].zeros();
+      }
+      else
+      {
+        C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) *
+            diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
+            eigvec.cols(0, negativeEigval(0) - 1).t();
+      }
+    }
+
+    // Output current objective function.
+    Info << "Active CMA-ES: iteration " << i << ", objective " << overallObjective
+        << "." << std::endl;
+
+    if (std::isnan(overallObjective) || std::isinf(overallObjective))
+    {
+      Warn << "Active CMA-ES: converged to " << overallObjective << "; "
+          << "terminating with failure.  Try a smaller step size?" << std::endl;
+
+      iterate = transformationPolicy.Transform(iterate);
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
+      return overallObjective;
+    }
+
+    if (std::abs(lastObjective - overallObjective) < tolerance)
+    {
+      if (steps > patience)
+      {
+        Info << "Active CMA-ES: minimized within tolerance " << tolerance << "; "
+            << "terminating optimization." << std::endl;
+
+        iterate = transformationPolicy.Transform(iterate);
+        Callback::EndOptimization(*this, function, iterate, callbacks...);
+        return overallObjective;
+      }
+    }
+    else
+    {
+      steps = 0;
+    }
+
+    steps++;
+    lastObjective = overallObjective;
+  }
+
+  iterate = transformationPolicy.Transform(iterate);
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
+  return overallObjective;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/cmaes/cmaes.hpp

@@ -17,6 +17,8 @@
 
 #include "full_selection.hpp"
 #include "random_selection.hpp"
+#include "transformation_policies/empty_transformation.hpp"
+#include "transformation_policies/boundary_box_constraint.hpp"
 
 namespace ens {
 
@@ -46,8 +48,12 @@ namespace ens {
  * ensmallen website.
  *
  * @tparam SelectionPolicy The selection strategy used for the evaluation step.
+ * @tparam TransformationPolicy The transformation strategy used to
+ *       map decision variables to the desired domain during fitness evaluation
+ *       and termination. Use EmptyTransformation if the domain isn't bounded.
  */
-template<typename SelectionPolicyType = FullSelection>
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
 class CMAES
 {
  public:
@@ -60,22 +66,24 @@ class CMAES
    * equal one pass over the dataset).
    *
    * @param lambda The population size (0 use the default size).
-   * @param lowerBound Lower bound of decision variables.
-   * @param upperBound Upper bound of decision variables.
+   * @param transformationPolicy Instantiated transformation policy used to
+   *     map the coordinates to the desired domain.
    * @param batchSize Batch size to use for the objective calculation.
    * @param maxIterations Maximum number of iterations allowed (0 means no
    *     limit).
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param selectionPolicy Instantiated selection policy used to calculate the
    *     objective.
+   * @param stepSize Starting sigma/step size (will be modified).
    */
   CMAES(const size_t lambda = 0,
-        const double lowerBound = -10,
-        const double upperBound = 10,
+        const TransformationPolicyType&
+              transformationPolicy = TransformationPolicyType(),
         const size_t batchSize = 32,
         const size_t maxIterations = 1000,
         const double tolerance = 1e-5,
-        const SelectionPolicyType& selectionPolicy = SelectionPolicyType());
+        const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+        double stepSize = 0);
 
   /**
    * Optimize the given function using CMA-ES. The given starting point will be
@@ -91,27 +99,17 @@ class CMAES
    * @return Objective value of the final point.
    */
   template<typename SeparableFunctionType,
-           typename MatType,
-           typename... CallbackTypes>
-  typename MatType::elem_type Optimize(SeparableFunctionType& function,
-                                       MatType& iterate,
-                                       CallbackTypes&&... callbacks);
+      typename MatType,
+      typename... CallbackTypes>
+      typename MatType::elem_type Optimize(SeparableFunctionType& function,
+          MatType& iterate,
+          CallbackTypes&&... callbacks);
 
-  //! Get the step size.
+  //! Get the population size.
   size_t PopulationSize() const { return lambda; }
-  //! Modify the step size.
+  //! Modify the population size.
   size_t& PopulationSize() { return lambda; }
 
-  //! Get the lower bound of decision variables.
-  double LowerBound() const { return lowerBound; }
-  //! Modify the lower bound of decision variables.
-  double& LowerBound() { return lowerBound; }
-
-  //! Get the upper bound of decision variables
-  double UpperBound() const { return upperBound; }
-  //! Modify the upper bound of decision variables
-  double& UpperBound() { return upperBound; }
-
   //! Get the batch size.
   size_t BatchSize() const { return batchSize; }
   //! Modify the batch size.
@@ -132,16 +130,25 @@ class CMAES
   //! Modify the selection policy.
   SelectionPolicyType& SelectionPolicy() { return selectionPolicy; }
 
+  //! Get the transformation policy.
+  const TransformationPolicyType& TransformationPolicy() const
+  { return transformationPolicy; }
+  //! Modify the transformation policy.
+  TransformationPolicyType& TransformationPolicy()
+  { return transformationPolicy; }
+
+  //! Get the step size.
+  double StepSize() const { return stepSize; }
+  //! Modify the step size.
+  double& StepSize() { return stepSize; }
+
+  //! Get the total number of function evaluations.
+  size_t FunctionEvaluations() const  { return functionEvaluations; }
+
  private:
   //! Population size.
   size_t lambda;
 
-  //! Lower bound of decision variables.
-  double lowerBound;
-
-  //! Upper bound of decision variables
-  double upperBound;
-
   //! The batch size for processing.
   size_t batchSize;
 
@@ -153,13 +160,25 @@ class CMAES
 
   //! The selection policy used to calculate the objective.
   SelectionPolicyType selectionPolicy;
+
+  //! The transformationPolicy used to map coordinates to the suitable domain
+  //! while evaluating fitness. This mapping is also done after optimization
+  //! has completed.
+  TransformationPolicyType transformationPolicy;
+
+  //! The step size.
+  double stepSize;
+
+  //! Counter for the number of function evaluations.
+  size_t functionEvaluations = 0;
 };
 
 /**
  * Convenient typedef for CMAES approximation.
  */
-template<typename SelectionPolicyType = RandomSelection>
-using ApproxCMAES = CMAES<SelectionPolicyType>;
+template<typename TransformationPolicyType = EmptyTransformation<>,
+         typename SelectionPolicyType = RandomSelection>
+using ApproxCMAES = CMAES<SelectionPolicyType, TransformationPolicyType>;
 
 } // namespace ens
 

include/ensmallen_bits/cmaes/cmaes_impl.hpp

@@ -22,36 +22,41 @@
 
 namespace ens {
 
-template<typename SelectionPolicyType>
-CMAES<SelectionPolicyType>::CMAES(const size_t lambda,
-                                  const double lowerBound,
-                                  const double upperBound,
+template<typename SelectionPolicyType, typename TransformationPolicyType>
+CMAES<SelectionPolicyType, TransformationPolicyType>::CMAES(const size_t lambda,
+                                  const TransformationPolicyType&
+                                        transformationPolicy,
                                   const size_t batchSize,
                                   const size_t maxIterations,
                                   const double tolerance,
-                                  const SelectionPolicyType& selectionPolicy) :
+                                  const SelectionPolicyType& selectionPolicy,
+                                  double stepSizeIn) :
     lambda(lambda),
-    lowerBound(lowerBound),
-    upperBound(upperBound),
     batchSize(batchSize),
     maxIterations(maxIterations),
     tolerance(tolerance),
-    selectionPolicy(selectionPolicy)
+    selectionPolicy(selectionPolicy),
+    transformationPolicy(transformationPolicy),
+    stepSize(stepSizeIn)
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
-template<typename SelectionPolicyType>
+template<typename SelectionPolicyType, typename TransformationPolicyType>
 template<typename SeparableFunctionType,
          typename MatType,
          typename... CallbackTypes>
-typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
+typename MatType::elem_type CMAES<SelectionPolicyType,
+  TransformationPolicyType>::Optimize(
     SeparableFunctionType& function,
     MatType& iterateIn,
     CallbackTypes&&... callbacks)
 {
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
-  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  typedef typename ForwardType<MatType>::bcol bcol;
+  typedef typename ForwardType<MatType>::uvec UVecType;
+  typedef typename ForwardType<MatType>::bmat BaseMatType;
 
   // Make sure that we have the methods that we need.  Long name...
   traits::CheckArbitrarySeparableFunctionTypeAPI<
@@ -69,16 +74,20 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
 
   // Parent weights.
   const size_t mu = std::round(lambda / 2);
-  BaseMatType w = std::log(mu + 0.5) - arma::log(
-      arma::linspace<BaseMatType>(0, mu - 1, mu) + 1.0);
-  w /= arma::accu(w);
+  BaseMatType w = std::log(mu + 0.5) - log(
+      linspace<BaseMatType>(0, mu - 1, mu) + 1.0);
+  w /= accu(w);
 
   // Number of effective solutions.
-  const double muEffective = 1 / arma::accu(arma::pow(w, 2));
+  const double muEffective = 1 / accu(pow(w, 2));
 
   // Step size control parameters.
   BaseMatType sigma(2, 1); // sigma is vector-shaped.
-  sigma(0) = 0.3 * (upperBound - lowerBound);
+  if (stepSize == 0)
+    sigma(0) = transformationPolicy.InitialStepSize();
+  else
+    sigma(0) = stepSize;
+
   const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5);
   const double ds = 1 + cs + 2 * std::max(std::sqrt((muEffective - 1) /
       (iterate.n_elem + 1)) - 1, 0.0);
@@ -99,23 +108,27 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
 
   std::vector<BaseMatType> mPosition(2, BaseMatType(iterate.n_rows,
       iterate.n_cols));
-  mPosition[0] = lowerBound + arma::randu<BaseMatType>(
-      iterate.n_rows, iterate.n_cols) * (upperBound - lowerBound);
+  mPosition[0] = iterate;
 
   BaseMatType step(iterate.n_rows, iterate.n_cols);
   step.zeros();
 
+  BaseMatType transformedIterate = transformationPolicy.Transform(iterate);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
   // Calculate the first objective function.
   ElemType currentObjective = 0;
   for (size_t f = 0; f < numFunctions; f += batchSize)
   {
     const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
-    const ElemType objective = function.Evaluate(mPosition[0], f,
+    const ElemType objective = function.Evaluate(transformedIterate, f,
         effectiveBatchSize);
     currentObjective += objective;
 
-    Callback::Evaluate(*this, function, mPosition[0], objective,
-        callbacks...);
+    terminate |= Callback::Evaluate(*this, function, transformedIterate,
+        objective, callbacks...);
   }
 
   ElemType overallObjective = currentObjective;
@@ -135,54 +148,65 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
   C[0].eye();
 
   // Covariance matrix parameters.
-  arma::Col<ElemType> eigval; // TODO: might need a more general type.
+  bcol eigval; // TODO: might need a more general type.
   BaseMatType eigvec;
   BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped.
   eigvalZero.zeros();
 
   // The current visitation order (sorted by population objectives).
-  arma::uvec idx = arma::linspace<arma::uvec>(0, lambda - 1, lambda);
+  UVecType idx = linspace<UVecType>(0, lambda - 1, lambda);
 
-  // Controls early termination of the optimization process.
-  bool terminate = false;
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
 
   // Now iterate!
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
+  Callback::BeginOptimization(*this, function, transformedIterate,
       callbacks...);
-  for (size_t i = 1; i < maxIterations && !terminate; ++i)
+
+  // The number of generations to wait after the minimum loss has
+  // been reached or no improvement has been made before terminating.
+  size_t patience = 10 + (30 * iterate.n_elem / lambda) + 1;
+  size_t steps = 0;
+
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     // To keep track of where we are.
-    const size_t idx0 = (i - 1) % 2;
-    const size_t idx1 = i % 2;
+    const size_t idx0 = i % 2;
+    const size_t idx1 = (i + 1) % 2;
 
     // Perform Cholesky decomposition. If the matrix is not positive definite,
     // add a small value and try again.
     BaseMatType covLower;
-    while (!arma::chol(covLower, C[idx0], "lower"))
-      C[idx0].diag() += 1e-16;
+    // while (!chol(covLower, C[idx0], "lower"))
+    while (!chol(covLower, C[idx0]))
+      C[idx0].diag() += std::numeric_limits<ElemType>::epsilon();
+
+    eig_sym(eigval, eigvec, C[idx0]);
 
     for (size_t j = 0; j < lambda; ++j)
     {
       if (iterate.n_rows > iterate.n_cols)
       {
-        pStep[idx(j)] = covLower *
-            arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
+        pStep[idx(j)] = covLower * BaseMatType(
+            iterate.n_rows, iterate.n_cols, GetFillType<MatType>::randn);
       }
       else
       {
-        pStep[idx(j)] = arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
-            * covLower;
+        pStep[idx(j)] = BaseMatType(
+          iterate.n_rows, iterate.n_cols, GetFillType<MatType>::randn) *
+          covLower.t();
       }
 
       pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];
 
       // Calculate the objective function.
       pObjective(idx(j)) = selectionPolicy.Select(function, batchSize,
-          pPosition[idx(j)], callbacks...);
+          transformationPolicy.Transform(pPosition[idx(j)]), terminate,
+          callbacks...);
     }
 
     // Sort population.
-    idx = arma::sort_index(pObjective);
+    idx = sort_index(pObjective);
 
     step = w(0) * pStep[idx(0)];
     for (size_t j = 1; j < mu; ++j)
@@ -192,7 +216,8 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
 
     // Calculate the objective function.
     currentObjective = selectionPolicy.Select(function, batchSize,
-        mPosition[idx1], callbacks...);
+        transformationPolicy.Transform(mPosition[idx1]), terminate,
+        callbacks...);
 
     // Update best parameters.
     if (currentObjective < overallObjective)
@@ -200,39 +225,54 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
       overallObjective = currentObjective;
       iterate = mPosition[idx1];
 
-      terminate |= Callback::StepTaken(*this, function, iterate, callbacks...);
+      transformedIterate = transformationPolicy.Transform(iterate);
+      terminate |= Callback::StepTaken(*this, function,
+          transformedIterate, callbacks...);
     }
 
     // Update Step Size.
     if (iterate.n_rows > iterate.n_cols)
     {
       ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
-          cs * (2 - cs) * muEffective) * covLower.t() * step;
+          cs * (2 - cs) * muEffective) * eigvec *
+          diagmat(1 / eigval) * eigvec.t() * step;
     }
     else
     {
       ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
-          cs * (2 - cs) * muEffective) * step * covLower.t();
+          cs * (2 - cs) * muEffective) * step * eigvec *
+          diagmat(1 / eigval) * eigvec.t();
     }
 
-    const ElemType psNorm = arma::norm(ps[idx1]);
-    sigma(idx1) = sigma(idx0) * std::exp(cs / ds * ( psNorm / enn - 1));
+    const ElemType psNorm = norm(ps[idx1]);
+    sigma(idx1) = sigma(idx0) * std::exp(cs / ds * (psNorm / enn - 1));
+
+    if (std::isnan(sigma(idx1)) || sigma(idx1) > 1e14)
+    {
+      Warn << "The step size diverged to " << sigma(idx1) << "; "
+          << "terminating with failure.  Try a smaller step size?" << std::endl;
+
+      iterate = transformationPolicy.Transform(iterate);
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
+      return overallObjective;
+    }
 
     // Update covariance matrix.
-    if ((psNorm / sqrt(1 - std::pow(1 - cs, 2 * i))) < h)
+    if ((psNorm / std::sqrt(1 - std::pow(1.0 - cs, 2.0 * (double) i))) < h)
     {
       pc[idx1] = (1 - cc) * pc[idx0] + std::sqrt(cc * (2 - cc) *
-        muEffective) * step;
+          muEffective) * step;
 
       if (iterate.n_rows > iterate.n_cols)
       {
         C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
-          (pc[idx1] * pc[idx1].t());
+            (pc[idx1] * pc[idx1].t());
       }
       else
       {
         C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
-          (pc[idx1].t() * pc[idx1]);
+            (pc[idx1].t() * pc[idx1]);
       }
     }
     else
@@ -255,21 +295,19 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
     {
       for (size_t j = 0; j < mu; ++j)
       {
-        C[idx1] = C[idx1] + cmu * w(j) *
-            pStep[idx(j)] * pStep[idx(j)].t();
+        C[idx1] = C[idx1] + cmu * w(j) * pStep[idx(j)] * pStep[idx(j)].t();
       }
     }
     else
     {
       for (size_t j = 0; j < mu; ++j)
       {
-        C[idx1] = C[idx1] + cmu * w(j) *
-            pStep[idx(j)].t() * pStep[idx(j)];
+        C[idx1] = C[idx1] + cmu * w(j) * pStep[idx(j)].t() * pStep[idx(j)];
       }
     }
 
-    arma::eig_sym(eigval, eigvec, C[idx1]);
-    const arma::uvec negativeEigval = arma::find(eigval < 0, 1);
+    eig_sym(eigval, eigvec, C[idx1]);
+    const UVecType negativeEigval = find(eigval < 0, 1);
     if (!negativeEigval.is_empty())
     {
       if (negativeEigval(0) == 0)
@@ -279,7 +317,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
       else
       {
         C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) *
-            arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
+            diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
             eigvec.cols(0, negativeEigval(0) - 1).t();
       }
     }
@@ -293,22 +331,32 @@ typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
       Warn << "CMA-ES: converged to " << overallObjective << "; "
           << "terminating with failure.  Try a smaller step size?" << std::endl;
 
+      iterate = transformationPolicy.Transform(iterate);
       Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
     if (std::abs(lastObjective - overallObjective) < tolerance)
     {
-      Info << "CMA-ES: minimized within tolerance " << tolerance << "; "
-          << "terminating optimization." << std::endl;
+      if (steps > patience) {
+        Info << "CMA-ES: minimized within tolerance " << tolerance << "; "
+            << "terminating optimization." << std::endl;
 
-      Callback::EndOptimization(*this, function, iterate, callbacks...);
-      return overallObjective;
+        iterate = transformationPolicy.Transform(iterate);
+        Callback::EndOptimization(*this, function, iterate, callbacks...);
+        return overallObjective;
+      }
     }
+    else {
+      steps = 0;
+    }
+
+    steps++;
 
     lastObjective = overallObjective;
   }
 
+  iterate = transformationPolicy.Transform(iterate);
   Callback::EndOptimization(*this, function, iterate, callbacks...);
   return overallObjective;
 }

include/ensmallen_bits/cmaes/full_selection.hpp

@@ -26,6 +26,7 @@ class FullSelection
    * @tparam SeparableFunctionType Type of the function to be evaluated.
    * @param function Function to optimize.
    * @param batchSize Batch size to use for each step.
+   * @param terminate Whether optimization should be terminated after this call.
    * @param iterate starting point.
    */
   template<typename SeparableFunctionType,
@@ -34,6 +35,7 @@ class FullSelection
   double Select(SeparableFunctionType& function,
                 const size_t batchSize,
                 const MatType& iterate,
+                bool& terminate,
                 CallbackTypes&... callbacks)
   {
     // Find the number of functions to use.
@@ -45,7 +47,8 @@ class FullSelection
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
       objective += function.Evaluate(iterate, f, effectiveBatchSize);
 
-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      terminate |= Callback::Evaluate(*this, f, iterate, objective,
+          callbacks...);
     }
 
     return objective;

include/ensmallen_bits/cmaes/pop_cmaes.hpp

@@ -0,0 +1,181 @@
+/**
+ * @file ipop_cmaes_impl.hpp
+ * @author Marcus Edel
+ * @author Benjami Parellada
+ *
+ * Definition of the IPOP Covariance Matrix Adaptation Evolution Strategy
+ * as proposed by A. Auger and N. Hansen in "A Restart CMA Evolution
+ * Strategy With Increasing Population Size" and BIPOP Covariance Matrix
+ * Adaptation Evolution Strategy as proposed by N. Hansen in "Benchmarking
+ * a BI-population CMA-ES on the BBOB-2009 function testbed".
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_POP_CMAES_HPP
+#define ENSMALLEN_CMAES_POP_CMAES_HPP
+
+#include "cmaes.hpp"
+
+namespace ens {
+
+/**
+ * Population-based CMA-ES (POP-CMA-ES) that can operate as either IPOP-CMA-ES
+ * or BIPOP-CMA-ES based on a flag.
+ *
+ * IPOP CMA-ES is a variant of the stochastic search algorithm
+ * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
+ * IPOP CMA-ES, also known as CMAES with increasing population size,
+ * incorporates a restart strategy that involves gradually increasing
+ * the population size. This approach is specifically designed to
+ * enhance the performance of CMA-ES on multi-modal functions.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @INPROCEEDINGS{1554902,
+ *   author    = {Auger, A. and Hansen, N.},
+ *   booktitle = {2005 IEEE Congress on Evolutionary Computation},
+ *   title     = {A restart CMA evolution strategy with increasing population
+ *                size},
+ *   year      = {2005},
+ *   volume    = {2},
+ *   number    = {},
+ *   pages     = {1769-1776 Vol. 2},
+ *   doi       = {10.1109/CEC.2005.1554902}}
+ * @endcode
+ *
+ * IPOP CMA-ES can optimize separable functions.  For more details, see the
+ * documentation on function types included with this distribution or on the
+ * ensmallen website.
+ *
+ * BI-Population CMA-ES is a variant of the stochastic search algorithm
+ * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
+ * It implements a dual restart strategy with varying population sizes: one
+ * increasing and one with smaller, varied sizes. This BI-population approach
+ * is designed to optimize performance on multi-modal function testbeds by
+ * leveraging different exploration and exploitation dynamics.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @inproceedings{hansen2009benchmarking,
+ *   title     = {Benchmarking a BI-population CMA-ES on the BBOB-2009 function
+ *                testbed},
+ *   author    = {Hansen, Nikolaus},
+ *   booktitle = {Proceedings of the 11th annual conference companion on genetic
+ *                and evolutionary computation conference: late breaking
+ *                papers},
+ *   pages     = {2389--2396},
+ *   year      = {2009}}
+ * @endcode
+ *
+ * BI-Population CMA-ES can efficiently handle separable, multimodal, and weak
+ * structure functions across various dimensions, as demonstrated in the
+ * comprehensive results of the BBOB-2009 function testbed. The optimizer
+ * utilizes an interlaced multistart strategy to balance between broad
+ * exploration and intensive exploitation, adjusting population sizes and
+ * step-sizes dynamically.
+ */
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>,
+         bool UseBIPOPFlag = true>
+class POP_CMAES : public CMAES<SelectionPolicyType, TransformationPolicyType>
+{
+ public:
+  /**
+   * Construct the POP-CMA-ES optimizer with the given parameters.
+   * Other than the same CMA-ES parameters, it also adds the maximum number of
+   * restarts, the increase in population factor, the maximum number of
+   * evaluations, as well as a flag indicating to use BIPOP or not.
+   * The suggested values are not necessarily good for the given problem, so it
+   * is suggested that the values used be tailored to the task at hand. The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param lambda The initial population size (0 use the default size).
+   * @param transformationPolicy Instantiated transformation policy used to
+   *    map the coordinates to the desired domain.
+   * @param batchSize Batch size to use for the objective calculation.
+   * @param maxIterations Maximum number of iterations allowed.
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param selectionPolicy Instantiated selection policy used to calculate the
+   *     objective.
+   * @param stepSize Starting sigma/step size (will be modified).
+   * @param populationFactor The factor by which population increases
+   *     after each restart.
+   * @param maxRestarts Maximum number of restarts.
+   * @param maxFunctionEvaluations Maximum number of function evaluations.
+   */
+  POP_CMAES(const size_t lambda = 0,
+            const TransformationPolicyType& transformationPolicy =
+                 TransformationPolicyType(),
+            const size_t batchSize = 32,
+            const size_t maxIterations = 1000,
+            const double tolerance = 1e-5,
+            const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+            double stepSize = 0,
+            const size_t maxRestarts = 9,
+            const double populationFactor = 2,
+            const size_t maxFunctionEvaluations = 1e9);
+
+  /**
+   * Set POP-CMA-ES specific parameters.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Get the population factor.
+  double PopulationFactor() const { return populationFactor; }
+  //! Modify the population factor.
+  double& PopulationFactor() { return populationFactor; }
+
+  //! Get the maximum number of restarts.
+  size_t MaxRestarts() const { return maxRestarts; }
+  //! Modify the maximum number of restarts.
+  size_t& MaxRestarts() { return maxRestarts; }
+
+  //! Get the maximum number of function evaluations.
+  size_t MaxFunctionEvaluations() const { return maxFunctionEvaluations; }
+  //! Modify the maximum number of function evaluations.
+  size_t& MaxFunctionEvaluations() { return maxFunctionEvaluations; }
+
+  //! Get the BIPOP mode flag.
+  static constexpr bool UseBIPOP() { return UseBIPOPFlag; }
+
+ private:
+  //! Population factor
+  double populationFactor;
+
+  //! Maximum number of restarts.
+  size_t maxRestarts;
+
+  //! Maximum number of function evaluations.
+  size_t maxFunctionEvaluations;
+
+};
+
+// Define IPOP_CMAES and BIPOP_CMAES using the POP_CMAES template
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
+using IPOP_CMAES = POP_CMAES<
+    SelectionPolicyType, TransformationPolicyType, false>;
+
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
+using BIPOP_CMAES = POP_CMAES<
+    SelectionPolicyType, TransformationPolicyType, true>;
+
+} // namespace ens
+
+// Include implementation.
+#include "pop_cmaes_impl.hpp"
+
+#endif

include/ensmallen_bits/cmaes/pop_cmaes_impl.hpp

@@ -0,0 +1,163 @@
+/**
+ * @file ipop_cmaes_impl.hpp
+ * @author Marcus Edel
+ * @author Benjami Parellada
+ *
+ * Implementation of the IPOP Covariance Matrix Adaptation Evolution Strategy
+ * as proposed by A. Auger and N. Hansen in "A Restart CMA Evolution
+ * Strategy With Increasing Population Size" and BIPOP Covariance Matrix
+ * Adaptation Evolution Strategy as proposed by N. Hansen in "Benchmarking
+ * a BI-population CMA-ES on the BBOB-2009 function testbed".
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_POP_CMAES_IMPL_HPP
+#define ENSMALLEN_CMAES_POP_CMAES_IMPL_HPP
+
+#include "pop_cmaes.hpp"
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+template<typename SelectionPolicyType, typename TransformationPolicyType, bool UseBIPOPFlag>
+POP_CMAES<SelectionPolicyType,
+          TransformationPolicyType,
+          UseBIPOPFlag>::POP_CMAES(
+    const size_t lambda,
+    const TransformationPolicyType& transformationPolicy,
+    const size_t batchSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const SelectionPolicyType& selectionPolicy,
+    double stepSize,
+    const size_t maxRestarts,
+    const double populationFactor,
+    const size_t maxFunctionEvaluations) :
+    CMAES<SelectionPolicyType, TransformationPolicyType>(
+        lambda, transformationPolicy, batchSize, maxIterations,
+        tolerance, selectionPolicy, stepSize),
+    populationFactor(populationFactor),
+    maxRestarts(maxRestarts),
+    maxFunctionEvaluations(maxFunctionEvaluations)
+{ /* Nothing to do. */ }
+
+template<typename SelectionPolicyType,
+         typename TransformationPolicyType,
+         bool UseBIPOPFlag>
+template<typename SeparableFunctionType, typename MatType, typename... CallbackTypes>
+typename MatType::elem_type POP_CMAES<SelectionPolicyType,
+    TransformationPolicyType, UseBIPOPFlag>::Optimize(
+    SeparableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+
+  StoreBestCoordinates<MatType> sbc;
+  StoreBestCoordinates<MatType> overallSBC;
+  size_t totalFunctionEvaluations = 0;
+  size_t largePopulationBudget = 0;
+  size_t smallPopulationBudget = 0;
+
+  // First single run with default population size
+  MatType iterate = iterateIn;
+  ElemType overallObjective = CMAES<SelectionPolicyType,
+      TransformationPolicyType>::Optimize(function, iterate, sbc,
+          callbacks...);
+
+  overallSBC = sbc;
+  ElemType objective;
+  size_t evaluations;
+
+  size_t defaultLambda = this->PopulationSize();
+  size_t currentLargeLambda = defaultLambda;
+
+  double stepSizeDefault = this->StepSize();
+
+  // Print out the default population size
+  Info << "Default population size: " << defaultLambda << "." << std::endl;
+
+  size_t restart = 0;
+
+  while (restart < maxRestarts)
+  {
+    if (!UseBIPOPFlag || largePopulationBudget <= smallPopulationBudget ||
+        restart == 0 || restart == maxRestarts - 1)
+    {
+      // Large population regime (IPOP or BIPOP)
+      currentLargeLambda *= populationFactor;
+      this->PopulationSize() = currentLargeLambda;
+      this->StepSize() = stepSizeDefault;
+
+      Info << "POP-CMA-ES: restart " << restart << ", large population size" <<
+          " (lambda): " << this->PopulationSize() << "." << std::endl;
+
+      iterate = iterateIn;
+
+      // Optimize using the CMAES object.
+      objective = CMAES<SelectionPolicyType,
+          TransformationPolicyType>::Optimize(function, iterate, sbc,
+          callbacks...);
+
+      evaluations = this->FunctionEvaluations();
+      largePopulationBudget += evaluations;
+    }
+    else if (UseBIPOPFlag)
+    {
+      // Small population regime (BIPOP only)
+      double u = arma::randu<double>();
+      size_t smallLambda = static_cast<size_t>(defaultLambda * std::pow(0.5 *
+          currentLargeLambda / defaultLambda, u * u));
+      double stepSizeSmall = 2 * std::pow(10, -2 * arma::randu<double>());
+
+      this->PopulationSize() = smallLambda;
+      this->StepSize() = stepSizeSmall;
+
+      Info << "BIPOP-CMA-ES: restart " << restart << ", small population" <<
+          " size (lambda): " << this->PopulationSize() << "." << std::endl;
+
+      iterate = iterateIn;
+
+      // Optimize using the CMAES object.
+      objective = CMAES<SelectionPolicyType,
+          TransformationPolicyType>::Optimize(function, iterate, sbc,
+                                              callbacks...);
+
+      evaluations = this->FunctionEvaluations();
+      smallPopulationBudget += evaluations;
+    }
+
+    if (objective < overallObjective)
+    {
+      overallObjective = objective;
+      overallSBC = sbc;
+      Info << "POP-CMA-ES: New best objective: " << overallObjective
+          << "." << std::endl;
+    }
+
+    totalFunctionEvaluations += evaluations;
+    // Check if the total number of evaluations has exceeded the limit
+    if (totalFunctionEvaluations >= maxFunctionEvaluations) {
+      Warn << "POP-CMA-ES: Maximum function overall evaluations reached. "
+           << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
+      iterateIn = std::move(overallSBC.BestCoordinates());
+      return overallSBC.BestObjective();
+    }
+
+    ++restart;
+  }
+
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
+  iterateIn = std::move(overallSBC.BestCoordinates());
+  return overallSBC.BestObjective();
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/cmaes/random_selection.hpp

@@ -41,6 +41,7 @@ class RandomSelection
    * @tparam SeparableFunctionType Type of the function to be evaluated.
    * @param function Function to optimize.
    * @param batchSize Batch size to use for each step.
+   * @param terminate Whether optimization should be terminated after this call.
    * @param iterate starting point.
    */
   template<typename SeparableFunctionType,
@@ -49,6 +50,7 @@ class RandomSelection
   double Select(SeparableFunctionType& function,
                 const size_t batchSize,
                 const MatType& iterate,
+                bool& terminate,
                 CallbackTypes&... callbacks)
   {
     // Find the number of functions to use.
@@ -64,7 +66,8 @@ class RandomSelection
 
       objective += function.Evaluate(iterate, selection, effectiveBatchSize);
 
-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      terminate |= Callback::Evaluate(*this, f, iterate, objective,
+          callbacks...);
     }
 
     return objective;

include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp

@@ -0,0 +1,162 @@
+/**
+ * @file boundary_box_constraint.hpp
+ * @author Suvarsha Chennareddy
+ *
+ * Boundary Box Transformation.
+ *
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_BOUNDARY_BOX_TRANSFORMATION_HPP
+#define ENSMALLEN_CMAES_BOUNDARY_BOX_TRANSFORMATION_HPP
+
+namespace ens {
+
+/**
+ * More often than not, coordinates must be bounded by some constraints.
+ * In a particular case, the domain of a specific function is restricted 
+ * by boundaries.
+ * The implemented transformation transforms given coordinates into a region 
+ * bounded by the given lower and upper bounds (a box). First, the 
+ * coordinates are shifted into a feasible preimage bounded by lowerBound - al
+ * and upperBound + au where al and au and calculated internally. 
+ * These shifted coordinates are then transformed into coordinates bounded by
+ * lower_bound and upper_bound. It is an identity transformation in between
+ * the lower and upper bounds.
+ * 
+ * For more information, check the original implementation in C by N. Hansen:
+ * https://github.com/CMA-ES/c-cmaes/blob/master/src/boundary_transformation.c
+ *
+ * @tparam MatType The matrix type of the coordinates and bounds.
+ */
+template<typename MatType = arma::mat>
+class BoundaryBoxConstraint
+{
+public:
+
+  /**
+   * Construct the boundary box constraint policy.
+   */
+  BoundaryBoxConstraint()
+  { /* Nothing to do. */ }
+
+  /**
+   * Construct the boundary box constraint policy.
+   *
+   * @param lowerBound The lower bound of the coordinates.
+   * @param upperBound The upper bound of the coordinates.
+   */
+  BoundaryBoxConstraint(const MatType& lowerBound,
+                        const MatType& upperBound) :
+      lowerBound(lowerBound),
+      upperBound(upperBound)
+  {}
+  
+  /**
+   * Construct the boundary box constraint policy.
+   *
+   * @param lowerBound The lower bound (for every dimension) of the coordinates.
+   * @param upperBound The upper bound (for every dimension) of the coordinates.
+   */
+  BoundaryBoxConstraint(const typename MatType::elem_type lowerBound,
+                        const typename MatType::elem_type upperBound) :
+      lowerBound({ (typename MatType::elem_type) lowerBound }),
+      upperBound({ (typename MatType::elem_type) upperBound })
+  {}
+
+  /**
+   * Map the given coordinates to the range 
+   * [lowerBound, upperBound]
+   *
+   * @param x Given coordinates.
+   * @return Transformed coordinates.
+   */
+  MatType Transform(const MatType& x)
+  {
+    typedef typename MatType::elem_type ElemType;
+    double diff, al, au, xlow, xup, r;
+    size_t Bi, Bj;
+    MatType y = x;
+    for (size_t i = 0; i < x.n_rows; i++)
+    {
+      Bi = (i < lowerBound.n_rows) ? i : (lowerBound.n_rows - 1);
+      for (size_t j = 0; j < x.n_cols; j++)
+      {
+        Bj = (j < lowerBound.n_cols) ? j : (lowerBound.n_cols - 1);
+
+        diff = (upperBound(Bi, Bj) - lowerBound(Bi, Bj)) / 2.0;
+        al = std::min(diff, (1 + std::abs(lowerBound(Bi, Bj))) / 20.0);
+        au = std::min(diff, (1 + std::abs(upperBound(Bi, Bj))) / 20.0);
+        xlow = lowerBound(Bi, Bj) - 2 * al - diff;
+        xup = upperBound(Bi, Bj) + 2 * au + diff;
+        r = 2 * (2 * diff + al + au);
+
+        // Shift y into feasible pre-image.
+        if (y(i, j) < xlow)
+        {
+          y(i,j) += (ElemType)(r * (1 + (int)((xlow - y(i, j)) / r)));
+        }
+        if (y(i, j) > xup)
+        {
+          y(i, j) -= (ElemType)(r * (1 + (int)((y(i, j) - xup) / r)));
+        }
+        if (y(i, j) < lowerBound(Bi, Bj) - al)
+        {
+          y(i, j) += (ElemType)(2 * (lowerBound(Bi, Bj) - al - y(i, j)));
+        }
+        if (y(i, j) > upperBound(Bi, Bj) + au)
+        {
+          y(i, j) -= (ElemType)(2 * (y(i, j) - upperBound(Bi, Bj) - au));
+        }
+
+        // Boundary transformation.
+        if (y(i, j) < lowerBound(Bi, Bj) + al)
+        {
+          y(i, j) = (ElemType)(lowerBound(Bi, Bj) +
+            (y(i, j) - (lowerBound(Bi, Bj) - al)) *
+            (y(i, j) - (lowerBound(Bi, Bj) - al)) / 4.0 / al);
+        }
+        else if (y(i,j) > upperBound(Bi,Bj) - au)
+        {
+          y(i, j) = (ElemType)(upperBound(Bi, Bj) -
+            (y(i, j) - (upperBound(Bi, Bj) + au)) *
+            (y(i, j) - (upperBound(Bi, Bj) + au)) / 4.0 / au);
+        }
+      }
+    }
+
+    return y;
+  }
+
+  /**
+   * Return a suitable initial step size.
+   *
+   * @return initial step size.
+   */
+  typename MatType::elem_type InitialStepSize()
+  { return 0.3 * (upperBound - lowerBound).min(); }
+
+  //! Get the lower bound of decision variables.
+  MatType LowerBound() const { return lowerBound; }
+  //! Modify the lower bound of decision variables.
+  MatType& LowerBound() { return lowerBound; }
+
+  //! Get the upper bound of decision variables.
+  MatType UpperBound() const { return upperBound; }
+  //! Modify the upper bound of decision variables.
+  MatType& UpperBound() { return upperBound; }
+
+private:
+  //! Lower bound of decision variables.
+  MatType lowerBound;
+
+  //! Upper bound of decision variables.
+  MatType upperBound;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp

@@ -0,0 +1,47 @@
+/**
+ * @file empty_transformation.hpp
+ * @author Suvarsha Chennareddy
+ *
+ * Empty Transformation, can also be called an Indentity Transformation.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP
+#define ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP
+
+namespace ens {
+
+/**
+ * This is an empty transformation. As the name indicates, it does
+ * not do anything. It is essentially an identity 
+ * transformation and is meant to be used when there are no
+ * sorts of constraints on the coordinates.
+ *
+ * @tparam MatType The matrix type of the coordinates.
+ */
+template<typename MatType = arma::mat>
+class EmptyTransformation
+{
+ public:
+  /**
+   * Transforms coordinates to themselves (effectively no transformation).
+   *
+   * @param x Input coordinates.
+   * @return Transformed coordinates (the coordinates themselves).
+   */
+  MatType Transform(const MatType& x) { return x; }
+
+  /**
+   * Return a suitable initial step size.
+   *
+   * @return initial step size.
+   */
+  typename MatType::elem_type InitialStepSize() { return 1; }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/cne/cne.hpp

@@ -149,14 +149,14 @@ class CNE
 
  private:
   //! Reproduce candidates to create the next generation.
-  template<typename MatType>
+  template<typename MatType, typename IndexType>
   void Reproduce(std::vector<MatType>& population,
                  const MatType& fitnessValues,
-                 arma::uvec& index);
+                 IndexType& index);
 
   //! Modify weights with some noise for the evolution of next generation.
-  template<typename MatType>
-  void Mutate(std::vector<MatType>& population, arma::uvec& index);
+  template<typename MatType, typename IndexType>
+  void Mutate(std::vector<MatType>& population, IndexType& index);
 
   /**
    * Crossover parents and create new childs. Two parents create two new childs.

include/ensmallen_bits/cne/cne_impl.hpp

@@ -47,6 +47,7 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
   typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename ForwardType<MatType>::uvec UVecType;
 
   // Make sure that we have the methods that we need.  Long name...
   traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType,
@@ -56,7 +57,7 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
   // Vector of fitness values corresponding to each candidate.
   BaseMatType fitnessValues;
   //! Index of sorted fitness values.
-  arma::uvec index;
+  UVecType index;
 
   // Make sure for evolution to work at least four candidates are present.
   if (populationSize < 4)
@@ -93,8 +94,8 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
   std::vector<BaseMatType> population;
   for (size_t i = 0 ; i < populationSize; ++i)
   {
-    population.push_back(arma::randn<BaseMatType>(iterate.n_rows,
-        iterate.n_cols) + iterate);
+    population.push_back(BaseMatType(iterate.n_rows, iterate.n_cols,
+        GetFillType<MatType>::randn) + iterate);
   }
 
   // Store the number of elements in the objective matrix.
@@ -111,26 +112,26 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
 
   // Find the fitness before optimization using given iterate parameters.
   ElemType lastBestFitness = function.Evaluate(iterate);
-  Callback::Evaluate(*this, function, iterate, lastBestFitness, callbacks...);
+  terminate |= Callback::Evaluate(*this, function, iterate, lastBestFitness,
+      callbacks...);
 
   // Iterate until maximum number of generations is obtained.
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
   for (size_t gen = 1; gen <= maxGenerations && !terminate; gen++)
   {
     // Calculating fitness values of all candidates.
     for (size_t i = 0; i < populationSize; i++)
     {
-       // Select a candidate and insert the parameters in the function.
-       iterate = population[i];
-       terminate |= Callback::StepTaken(*this, function, iterate,
-          callbacks...);
+        // Select a candidate and insert the parameters in the function.
+        iterate = population[i];
+        terminate |= Callback::StepTaken(*this, function, iterate,
+            callbacks...);
 
-       // Find fitness of candidate.
-       fitnessValues[i] = function.Evaluate(iterate);
+        // Find fitness of candidate.
+        fitnessValues[i] = function.Evaluate(iterate);
 
-       Callback::Evaluate(*this, function, iterate, fitnessValues[i],
-          callbacks...);
+        terminate |= Callback::Evaluate(*this, function, iterate,
+            fitnessValues[i], callbacks...);
     }
 
     Info << "Generation number: " << gen << " best fitness = "
@@ -154,21 +155,23 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
   // Set the best candidate into the network parameters.
   iterateIn = population[index(0)];
 
+  // The output of the callback doesn't matter because the optimization is
+  // finished.
   const ElemType objective = function.Evaluate(iterate);
-  Callback::Evaluate(*this, function, iterate, objective, callbacks...);
+  (void) Callback::Evaluate(*this, function, iterate, objective, callbacks...);
 
   Callback::EndOptimization(*this, function, iterate, callbacks...);
   return objective;
 }
 
 //! Reproduce candidates to create the next generation.
-template<typename MatType>
+template<typename MatType, typename IndexType>
 inline void CNE::Reproduce(std::vector<MatType>& population,
                            const MatType& fitnessValues,
-                           arma::uvec& index)
+                           IndexType& index)
 {
   // Sort fitness values. Smaller fitness value means better performance.
-  index = arma::sort_index(fitnessValues);
+  index = sort_index(fitnessValues);
 
   // First parent.
   size_t mom;
@@ -239,17 +242,20 @@ inline void CNE::Crossover(std::vector<MatType>& population,
 }
 
 //! Modify weights with some noise for the evolution of next generation.
-template<typename MatType>
-inline void CNE::Mutate(std::vector<MatType>& population, arma::uvec& index)
+template<typename MatType, typename IndexType>
+inline void CNE::Mutate(std::vector<MatType>& population, IndexType& index)
 {
+  typedef typename MatType::elem_type ElemType;
+
   // Mutate the whole matrix with the given rate and probability.
   // The best candidate is not altered.
   for (size_t i = 1; i < populationSize; i++)
   {
-    population[index(i)] += (arma::randu<MatType>(population[index(i)].n_rows,
-        population[index(i)].n_cols) < mutationProb) %
-        (mutationSize * arma::randn<MatType>(population[index(i)].n_rows,
-        population[index(i)].n_cols));
+    population[index(i)] += conv_to<MatType>::from(
+        randu<MatType>(population[index(i)].n_rows,
+        population[index(i)].n_cols) < ElemType(mutationProb)) %
+        (ElemType(mutationSize) * MatType(population[index(i)].n_rows,
+        population[index(i)].n_cols, GetFillType<MatType>::randn));
   }
 }
 

include/ensmallen_bits/config.hpp

@@ -56,18 +56,8 @@
 #endif
 
 
-// Define ens_deprecated for deprecated functionality.
-// This is adapted from Armadillo's implementation.
-#if defined(_MSC_VER)
-  #define ens_deprecated __declspec(deprecated)
-#elif defined(__GNUG__) && (!defined(__clang__))
-  #define ens_deprecated __attribute__((__deprecated__))
-#elif defined(__clang__)
-  #if __has_attribute(__deprecated__)
-    #define ens_deprecated __attribute__((__deprecated__))
-  #else
-    #define ens_deprecated
-  #endif
-#else
-  #define ens_deprecated
+// undefine conflicting macros
+#if defined(As)
+  #pragma message ("WARNING: undefined conflicting 'As' macro")
+  #undef As
 #endif

include/ensmallen_bits/de/de.hpp

@@ -45,10 +45,10 @@ namespace ens {
  *
  * @code
  * @techreport{storn1995,
- *   title    = {Differential Evolution—a simple and efficient adaptive scheme
- *               for global optimization over continuous spaces},
- *   author   = {Storn, Rainer and Price, Kenneth},
- *   year     = 1995
+ *   title  = {Differential Evolution—a simple and efficient adaptive scheme
+ *             for global optimization over continuous spaces},
+ *   author = {Storn, Rainer and Price, Kenneth},
+ *   year   = 1995
  * }
  * @endcode
  *

include/ensmallen_bits/de/de_impl.hpp

@@ -40,14 +40,16 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
   typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename ForwardType<MatType>::vec ColType;
 
   BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // Population matrix. Each column is a candidate.
   std::vector<BaseMatType> population;
   population.resize(populationSize);
+
   // Vector of fitness values corresponding to each candidate.
-  arma::Col<ElemType> fitnessValues;
+  ColType fitnessValues;
 
   // Make sure that we have the methods that we need.  Long name...
   traits::CheckArbitraryFunctionTypeAPI<
@@ -57,13 +59,13 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
   // Population Size must be at least 3 for DE to work.
   if (populationSize < 3)
   {
-    throw std::logic_error("CNE::Optimize(): population size should be at least"
+    throw std::logic_error("DE::Optimize(): population size should be at least"
         " 3!");
   }
 
   // Initialize helper variables.
   fitnessValues.set_size(populationSize);
-  ElemType lastBestFitness = DBL_MAX;
+  ElemType lastBestFitness = std::numeric_limits<ElemType>::max();
   BaseMatType bestElement;
 
   // Controls early termination of the optimization process.
@@ -77,19 +79,18 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
     population[i] += iterate;
     fitnessValues[i] = function.Evaluate(population[i]);
 
-    Callback::Evaluate(*this, function, population[i], fitnessValues[i],
-        callbacks...);
+    terminate |= Callback::Evaluate(*this, function, population[i],
+        fitnessValues[i], callbacks...);
 
     if (fitnessValues[i] < lastBestFitness)
     {
-      lastBestFitness = fitnessValues[i];
+      lastBestFitness = ElemType(fitnessValues[i]);
       bestElement = population[i];
     }
   }
 
   // Iterate until maximum number of generations are completed.
-  terminate |= Callback::BeginOptimization(*this, function, iterate,
-      callbacks...);
+  Callback::BeginOptimization(*this, function, iterate, callbacks...);
   for (size_t gen = 0; gen < maxGenerations && !terminate; gen++)
   {
     // Generate new population based on /best/1/bin strategy.
@@ -112,24 +113,30 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
       while (m == member && m == l);
 
       // Generate new "mutant" from two randomly chosen members.
-      BaseMatType mutant = bestElement + differentialWeight *
+      BaseMatType mutant = bestElement + ElemType(differentialWeight) *
           (population[l] - population[m]);
 
       // Perform crossover.
-      const BaseMatType cr = arma::randu<BaseMatType>(iterate.n_rows);
+      BaseMatType cr;
+      cr.randu(iterate.n_rows, 1);
       for (size_t it = 0; it < iterate.n_rows; it++)
       {
-        if (cr[it] >= crossoverRate)
+        if (cr[it] >= ElemType(crossoverRate))
         {
-          mutant[it] = iterate[it];
+          mutant(it) = ElemType(iterate(it));
         }
       }
 
       ElemType iterateValue = function.Evaluate(iterate);
-      Callback::Evaluate(*this, function, iterate, iterateValue, callbacks...);
+      terminate |= Callback::Evaluate(*this, function, iterate, iterateValue,
+          callbacks...);
 
       const ElemType mutantValue = function.Evaluate(mutant);
-      Callback::Evaluate(*this, function, mutant, mutantValue, callbacks...);
+      terminate |= Callback::Evaluate(*this, function, mutant, mutantValue,
+          callbacks...);
+
+      if (terminate)
+        break;
 
       // Replace the current member if mutant is better.
       if (mutantValue < iterateValue)
@@ -154,7 +161,7 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
     }
 
     // Update helper variables.
-    lastBestFitness = fitnessValues.min();
+    lastBestFitness = ElemType(fitnessValues.min());
     for (size_t it = 0; it < populationSize; it++)
     {
       if (fitnessValues[it] == lastBestFitness)

include/ensmallen_bits/delta_bar_delta/delta_bar_delta.hpp

@@ -0,0 +1,179 @@
+/**
+ * @file delta_bar_delta.hpp
+ * @author Ranjodh Singh
+ *
+ * Class wrapper for the DeltaBarDelta update policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DELTA_BAR_DELTA_HPP
+#define ENSMALLEN_DELTA_BAR_DELTA_HPP
+
+#include <ensmallen_bits/gradient_descent/gradient_descent.hpp>
+#include "update_policies/delta_bar_delta_update.hpp"
+
+namespace ens {
+
+/**
+ * DeltaBarDelta optimizer.
+ *
+ * A heuristic designed to accelerate convergence by
+ * adapting the learning rate of each parameter individually.
+ *
+ * According to the Delta-Bar-Delta update:
+ *
+ * - If the current gradient and the exponential average of
+ *   past gradients corresponding to a parameter have the same
+ *   sign, then the step size for that parameter is incremented by
+ *   \f$\kappa\f$. Otherwise, it is decreased by a proportion \f$\phi\f$
+ *   of its current value (additive increase, multiplicative decrease).
+ *
+ * @note This implementation uses a minStepSize parameter to set a lower
+ *     bound for the learning rate. This prevents the learning rate from
+ *     dropping to zero, which can occur due to floating-point underflow.
+ *     For tasks which require extreme fine-tuning, you may need to lower
+ *     this parameter below its default value (1e-8) in order to allow for
+ *     smaller learning rates.
+ *
+ * @code
+ * @article{jacobs1988increased,
+ *   title     = {Increased Rates of Convergence Through Learning Rate
+ *                Adaptation},
+ *   author    = {Jacobs, Robert A.},
+ *   journal   = {Neural Networks},
+ *   volume    = {1},
+ *   number    = {4},
+ *   pages     = {295--307},
+ *   year      = {1988},
+ *   publisher = {Pergamon}
+ * }
+ * @endcode
+ */
+class DeltaBarDelta
+{
+ public:
+  /**
+   * Construct the DeltaBarDelta optimizer with the given function and
+   * parameters. DeltaBarDelta is very sensitive to its parameters (kappa
+   * and phi) hence a good hyperparameter selection is necessary as its
+   * default may not fit every case.
+   *
+   * @param stepSize Step size (initial).
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param kappa Additive increase constant for step size.
+   * @param phi Multiplicative decrease factor for step size.
+   * @param theta Decay rate for the exponential moving average.
+   * @param minStepSize Minimum allowed step size for any parameter
+   *     (default: 1e-8).
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   */
+  DeltaBarDelta(const double stepSize = 1.0,
+                const size_t maxIterations = 100000,
+                const double tolerance = 1e-5,
+                const double kappa = 0.2,
+                const double phi = 0.2,
+                const double theta = 0.5,
+                const double minStepSize = 1e-8,
+                const bool resetPolicy = true);
+
+  /**
+   * Optimize the given function using DeltaBarDelta.
+   * The given starting point will be modified to store the finishing
+   * point of the algorithm, and the final objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the initial step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the initial step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the additive increase constant for step size.
+  double Kappa() const { return optimizer.UpdatePolicy().Kappa(); }
+  //! Modify the additive increase constant for step size.
+  double& Kappa() { return optimizer.UpdatePolicy().Kappa(); }
+
+  //! Get the multiplicative decrease factor for step size.
+  double Phi() const { return optimizer.UpdatePolicy().Phi(); }
+  //! Modify the multiplicative decrease factor for step size.
+  double& Phi() { return optimizer.UpdatePolicy().Phi(); }
+
+  //! Get the decay rate for the exponential moving average.
+  double Theta() const { return optimizer.UpdatePolicy().Theta(); }
+  //! Modify the decay rate for the exponential moving average.
+  double& Theta() { return optimizer.UpdatePolicy().Theta(); }
+
+  //! Get the minimum allowed step size for any parameter.
+  double MinStepSize() const { return optimizer.UpdatePolicy().MinStepSize(); }
+  //! Modify the minimum allowed step size for any parameter.
+  double& MinStepSize() { return optimizer.UpdatePolicy().MinStepSize(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The GradientDescentType object with DeltaBarDelta policy.
+  GradientDescentType<DeltaBarDeltaUpdate, NoDecay> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "delta_bar_delta_impl.hpp"
+
+#endif // ENSMALLEN_DELTA_BAR_DELTA_HPP

include/ensmallen_bits/delta_bar_delta/delta_bar_delta_impl.hpp

@@ -0,0 +1,41 @@
+/**
+ * @file delta_bar_delta_impl.hpp
+ * @author Ranjodh Singh
+ *
+ * Implementation of DeltaBarDelta class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DELTA_BAR_DELTA_IMPL_HPP
+#define ENSMALLEN_DELTA_BAR_DELTA_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "./delta_bar_delta.hpp"
+
+namespace ens {
+
+inline DeltaBarDelta::DeltaBarDelta(
+    const double stepSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const double kappa,
+    const double phi,
+    const double theta,
+    const double minStepSize,
+    const bool resetPolicy) : 
+    optimizer(stepSize,
+              maxIterations,
+              tolerance,
+              DeltaBarDeltaUpdate(stepSize, kappa, phi, theta, minStepSize),
+              NoDecay(),
+              resetPolicy)
+{
+  /* Nothing to do. */
+}
+
+} // namespace ens
+
+#endif // ENSMALLEN_DELTA_BAR_DELTA_IMPL_HPP

include/ensmallen_bits/delta_bar_delta/momentum_delta_bar_delta.hpp

@@ -0,0 +1,183 @@
+/**
+ * @file momentum_delta_bar_delta.hpp
+ * @author Ranjodh Singh
+ *
+ * Class wrapper for the MomentumDeltaBarDelta update policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_HPP
+#define ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_HPP
+
+#include <ensmallen_bits/gradient_descent/gradient_descent.hpp>
+#include "update_policies/momentum_delta_bar_delta_update.hpp"
+
+namespace ens {
+
+/**
+ * MomentumDeltaBarDelta Optimizer.
+ *
+ * A DeltaBarDelta variant that incorporates the following modifications:
+ *  - In the original DeltaBarDelta, the momentum term (delta_bar) is used
+ *    solely for sign comparison with the current gradient and does not
+ *    participate in the parameter update. In this modified variant, the
+ *    momentum term (velocity) is directly used to update the parameters.
+ *  - Instead of adjusting the step size directly, each parameter maintains
+ *    a gain value initialized to 1.0. Updates apply additive increases or
+ *    multiplicative decreases to this gain. The effective step size for a
+ *    parameter is the product of its initial step size and its current gain.
+ *
+ * Note: This variant originates from optimization of the t-SNE cost function.
+ *
+ * @code
+ * @article{maaten2008visualizing,
+ *   title={Visualizing data using t-SNE},
+ *   author={van der Maaten, Laurens and Hinton, Geoffrey},
+ *   journal={Journal of machine learning research},
+ *   volume={9},
+ *   pages={2579--2605},
+ *   month={11},
+ *   year={2008}
+ * }
+ * @endcode
+ *
+ * @code
+ * @article{jacobs1988increased,
+ *   title     = {Increased Rates of Convergence Through Learning Rate
+ *                Adaptation},
+ *   author    = {Jacobs, Robert A.},
+ *   journal   = {Neural Networks},
+ *   volume    = {1},
+ *   number    = {4},
+ *   pages     = {295--307},
+ *   year      = {1988},
+ *   publisher = {Pergamon}
+ * }
+ * @endcode
+ */
+class MomentumDeltaBarDelta
+{
+ public:
+  /**
+   * Construct the MomentumDeltaBarDelta optimizer with the given function and
+   * parameters.
+   *
+   * @param stepSize Step size (initial).
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param kappa Additive increase constant for step size.
+   * @param phi Multiplicative decrease factor for step size.
+   * @param momentum The momentum decay hyperparameter.
+   * @param minGain Minimum allowed gain (scaling factor) for any parameter
+   *     (default: 1e-8).
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   */
+  MomentumDeltaBarDelta(const double stepSize = 1.0,
+                        const size_t maxIterations = 100000,
+                        const double tolerance = 1e-5,
+                        const double kappa = 0.2,
+                        const double phi = 0.8,
+                        const double momentum = 0.5,
+                        const double minGain = 1e-8,
+                        const bool resetPolicy = true);
+
+  /**
+   * Optimize the given function using MomentumDeltaBarDelta.
+   * The given starting point will be modified to store the finishing
+   * point of the algorithm, and the final objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the initial step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the initial step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the additive increase constant for step size.
+  double Kappa() const { return optimizer.UpdatePolicy().Kappa(); }
+  //! Modify the additive increase constant for step size.
+  double& Kappa() { return optimizer.UpdatePolicy().Kappa(); }
+
+  //! Get the multiplicative decrease factor for step size.
+  double Phi() const { return optimizer.UpdatePolicy().Phi(); }
+  //! Modify the multiplicative decrease factor for step size.
+  double& Phi() { return optimizer.UpdatePolicy().Phi(); }
+
+  //! Get the momentum decay hyperparameter.
+  double Momentum() const { return optimizer.UpdatePolicy().Momentum(); }
+  //! Modify the momentum decay hyperparameter.
+  double& Momentum() { return optimizer.UpdatePolicy().Momentum(); }
+
+  //! Get the minimum allowed gain (scaling factor) for any parameter.
+  double MinGain() const { return optimizer.UpdatePolicy().MinGain(); }
+  //! Modify the minimum allowed gain (scaling factor) for any parameter.
+  double& MinGain() { return optimizer.UpdatePolicy().MinGain(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The GradientDescentType object with MomentumDeltaBarDelta policy.
+  GradientDescentType<MomentumDeltaBarDeltaUpdate, NoDecay> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "momentum_delta_bar_delta_impl.hpp"
+
+#endif // ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_HPP

include/ensmallen_bits/delta_bar_delta/momentum_delta_bar_delta_impl.hpp

@@ -0,0 +1,41 @@
+/**
+ * @file momentum_delta_bar_delta_impl.hpp
+ * @author Ranjodh Singh
+ *
+ * Implementation of MomentumDeltaBarDelta class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP
+#define ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "./momentum_delta_bar_delta.hpp"
+
+namespace ens {
+
+inline MomentumDeltaBarDelta::MomentumDeltaBarDelta(
+    const double stepSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const double kappa,
+    const double phi,
+    const double momentum,
+    const double minGain,
+    const bool resetPolicy) : 
+    optimizer(stepSize,
+              maxIterations,
+              tolerance,
+              MomentumDeltaBarDeltaUpdate(kappa, phi, momentum, minGain),
+              NoDecay(),
+              resetPolicy)
+{
+  /* Nothing to do. */
+}
+
+} // namespace ens
+
+#endif // ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP

include/ensmallen_bits/delta_bar_delta/update_policies/delta_bar_delta_update.hpp

@@ -0,0 +1,197 @@
+/**
+ * @file delta_bar_delta_update.hpp
+ * @author Ranjodh Singh
+ *
+ * DeltaBarDelta update policy for Gradient Descent.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DELTA_BAR_DELTA_UPDATE_HPP
+#define ENSMALLEN_DELTA_BAR_DELTA_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * DeltaBarDelta update policy for Gradient Descent.
+ *
+ * A heuristic designed to accelerate convergence by
+ * adapting the learning rate of each parameter individually.
+ *
+ * According to the Delta-Bar-Delta update:
+ *
+ * - If the current gradient and the exponential average of
+ *   past gradients corresponding to a parameter have the same
+ *   sign, then the step size for that parameter is incremented by
+ *   \f$\kappa\f$. Otherwise, it is decreased by a proportion \f$\phi\f$
+ *   of its current value (additive increase, multiplicative decrease).
+ *
+ * @note This implementation uses a minStepSize parameter to set a lower
+ *     bound for the learning rate. This prevents the learning rate from
+ *     dropping to zero, which can occur due to floating-point underflow.
+ *     For tasks which require extreme fine-tuning, you may need to lower
+ *     this parameter below its default value (1e-8) in order to allow for
+ *     smaller learning rates.
+ *
+ * @code
+ * @article{jacobs1988increased,
+ *   title     = {Increased Rates of Convergence Through Learning Rate
+ *                Adaptation},
+ *   author    = {Jacobs, Robert A.},
+ *   journal   = {Neural Networks},
+ *   volume    = {1},
+ *   number    = {4},
+ *   pages     = {295--307},
+ *   year      = {1988},
+ *   publisher = {Pergamon}
+ * }
+ * @endcode
+ */
+class DeltaBarDeltaUpdate
+{
+ public:
+  /**
+   * Construct the DeltaBarDelta update policy with given parameters.
+   *
+   * @param initialStepSize Initial Step Size.
+   * @param kappa Additive increase constant for step size.
+   * @param phi Multiplicative decrease factor for step size.
+   * @param theta Decay rate for the exponential moving average.
+   * @param minStepSize Minimum allowed step size for any parameter
+   *     (default: 1e-8).
+   */
+  DeltaBarDeltaUpdate(
+      const double initialStepSize,
+      const double kappa,
+      const double phi,
+      const double theta,
+      const double minStepSize = 1e-8) :
+      initialStepSize(initialStepSize),
+      kappa(kappa),
+      phi(phi),
+      theta(theta),
+      minStepSize(minStepSize)
+  {
+    /* Do nothing. */
+  }
+
+  //! Access the initialStepSize hyperparameter.
+  double InitialStepSize() const { return initialStepSize; }
+  //! Modify the initialStepSize hyperparameter.
+  double& InitialStepSize() { return initialStepSize; }
+
+  //! Access the kappa hyperparameter.
+  double Kappa() const { return kappa; }
+  //! Modify the kappa hyperparameter.
+  double& Kappa() { return kappa; }
+
+  //! Access the phi hyperparameter.
+  double Phi() const { return phi; }
+  //! Modify the phi hyperparameter.
+  double& Phi() { return phi; }
+
+  //! Access the theta hyperparameter.
+  double Theta() const { return theta; }
+  //! Modify the theta hyperparameter.
+  double& Theta() { return theta; }
+
+  //! Access the minStepSize hyperparameter.
+  double MinStepSize() const { return minStepSize; }
+  //! Modify the minStepSize hyperparameter.
+  double& MinStepSize() { return minStepSize; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType. This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template <typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    typedef typename MatType::elem_type ElemType;
+
+    /**
+     * This is called by the optimizer method before the start of the iteration
+     * update process.
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(
+        const DeltaBarDeltaUpdate& parent,
+        const size_t rows,
+        const size_t cols) :
+        parent(parent),
+        kappa(ElemType(parent.kappa)),
+        phi(ElemType(parent.phi)),
+        theta(ElemType(parent.theta)),
+        minStepSize(ElemType(parent.minStepSize))
+    {
+      deltaBar.zeros(rows, cols);
+      epsilon.set_size(rows, cols);
+      epsilon.fill(ElemType(parent.InitialStepSize()));
+    }
+
+    /**
+     * Update step for Gradient Descent.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param delta The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double /* stepSize */,
+                const GradType& delta)
+    {
+      const MatType signMatrix = sign(delta % deltaBar);
+
+      epsilon += conv_to<MatType>::from((signMatrix == +1) * kappa -
+          (signMatrix == -1) * phi % epsilon);
+      epsilon.clamp(minStepSize, arma::Datum<ElemType>::inf);
+
+      deltaBar = theta * deltaBar + (1 - theta) * delta;
+      iterate -= epsilon % delta;
+    }
+
+   private:
+    //! The instantiated parent class.
+    const DeltaBarDeltaUpdate& parent;
+
+    //! The exponential average of past gradients.
+    MatType deltaBar;
+    
+    //! Tracks the current step size for each parameter.
+    MatType epsilon;
+
+    // Parent parameters converted to the element type of the matrix.
+    ElemType kappa;
+    ElemType phi;
+    ElemType theta;
+    ElemType minStepSize;
+  };
+
+ private:
+  //! The initialStepSize hyperparameter.
+  double initialStepSize;
+
+  //! The kappa hyperparameter.
+  double kappa;
+
+  //! The phi hyperparameter.
+  double phi;
+
+  //! The theta hyperparameter.
+  double theta;
+
+  //! The minStepSize hyperparameter.
+  double minStepSize;
+};
+
+} // namespace ens
+
+#endif // ENSMALLEN_DELTA_BAR_DELTA_UPDATE_HPP

include/ensmallen_bits/delta_bar_delta/update_policies/momentum_delta_bar_delta_update.hpp

@@ -0,0 +1,178 @@
+/**
+ * @file momentum_delta_bar_delta_update.hpp
+ * @author Ranjodh Singh
+ *
+ * MomentumDeltaBarDelta update policy for Gradient Descent.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_UPDATE_HPP
+#define ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * MomentumDeltaBarDelta update policy for Gradient Descent.
+ *
+ * A DeltaBarDelta variant that incorporates the following modifications:
+ *  - In the original DeltaBarDelta, the momentum term (delta_bar) is used
+ *    solely for sign comparison with the current gradient and does not
+ *    participate in the parameter update. In this modified variant, the
+ *    momentum term (velocity) is directly used to update the parameters.
+ *  - Instead of adjusting the step size directly, each parameter maintains
+ *    a gain value initialized to 1.0. Updates apply additive increases or
+ *    multiplicative decreases to this gain. The effective step size for a
+ *    parameter is the product of its initial step size and its current gain.
+ *
+ * Note: This variant originates from optimization of the t-SNE cost function.
+ *
+ * @code
+ * @article{jacobs1988increased,
+ *   title     = {Increased Rates of Convergence Through Learning Rate
+ *                Adaptation},
+ *   author    = {Jacobs, Robert A.},
+ *   journal   = {Neural Networks},
+ *   volume    = {1},
+ *   number    = {4},
+ *   pages     = {295--307},
+ *   year      = {1988},
+ *   publisher = {Pergamon}
+ * }
+ * @endcode
+ */
+class MomentumDeltaBarDeltaUpdate
+{
+ public:
+  /**
+   * Construct the MomentumDeltaBarDelta update policy with given parameters.
+   *
+   * @param kappa Additive increase constant for step size.
+   * @param phi Multiplicative decrease factor for step size.
+   * @param momentum The momentum decay hyperparameter.
+   * @param minGain Minimum allowed gain (scaling factor) for any parameter
+   *     (default: 1e-8).
+   */
+  MomentumDeltaBarDeltaUpdate(
+      const double kappa = 0.2,
+      const double phi = 0.8,
+      const double momentum = 0.5,
+      const double minGain = 1e-8) :
+      kappa(kappa),
+      phi(phi),
+      momentum(momentum),
+      minGain(minGain)
+  {
+    /* Do nothing. */
+  }
+
+  //! Access the kappa hyperparameter.
+  double Kappa() const { return kappa; }
+  //! Modify the kappa hyperparameter.
+  double& Kappa() { return kappa; }
+
+  //! Access the phi hyperparameter.
+  double Phi() const { return phi; }
+  //! Modify the phi hyperparameter.
+  double& Phi() { return phi; }
+
+  //! Access the momentum hyperparameter.
+  double Momentum() const { return momentum; }
+  //! Modify the momentum hyperparameter.
+  double& Momentum() { return momentum; }
+
+  //! Access the minGain hyperparameter.
+  double MinGain() const { return minGain; }
+  //! Modify the minGain hyperparameter.
+  double& MinGain() { return minGain; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType. This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template <typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    typedef typename MatType::elem_type ElemType;
+
+    /**
+     * This is called by the optimizer method before the start of the iteration
+     * update process.
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(
+        const MomentumDeltaBarDeltaUpdate& parent,
+        const size_t rows,
+        const size_t cols) :
+        parent(parent),
+        kappa(ElemType(parent.kappa)),
+        phi(ElemType(parent.phi)),
+        momentum(ElemType(parent.momentum)),
+        minGain(ElemType(parent.minGain))
+    {
+      gains.ones(rows, cols);
+      velocity.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for Gradient Descent.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      gains += conv_to<MatType>::from(
+          (sign(gradient) != sign(velocity)) * kappa -
+          (sign(gradient) == sign(velocity)) * (1 - phi) % gains);
+      gains.clamp(minGain, arma::Datum<ElemType>::inf);
+
+      velocity = momentum * velocity - (ElemType(stepSize) * gains) % gradient;
+      iterate += velocity;
+    }
+
+   private:
+    //! The instantiated parent class.
+    const MomentumDeltaBarDeltaUpdate& parent;
+
+    //! The gains matrix.
+    MatType gains;
+
+    //! The velocity matrix.
+    MatType velocity;
+
+    // Parent parameters converted to the element type of the matrix.
+    ElemType kappa;
+    ElemType phi;
+    ElemType momentum;
+    ElemType minGain;
+  };
+
+ private:
+  //! The kappa hyperparameter.
+  double kappa;
+
+  //! The phi hyperparameter.
+  double phi;
+
+  //! The momentum hyperparameter.
+  double momentum;
+
+  //! The minGain hyperparameter.
+  double minGain;
+};
+
+} // namespace ens
+
+#endif // ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_UPDATE_HPP

include/ensmallen_bits/demon_adam/demon_adam.hpp

@@ -0,0 +1,207 @@
+/**
+ * @file demon_adam.hpp
+ * @author Marcus Edel
+ *
+ * Definition of DemonAdam.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_ADAM_DEMON_ADAM_HPP
+#define ENSMALLEN_DEMON_ADAM_DEMON_ADAM_HPP
+
+#include "../sgd/sgd.hpp"
+#include "../adam/adam_update.hpp"
+#include "../adam/adamax_update.hpp"
+#include "../adam/amsgrad_update.hpp"
+#include "../adam/nadam_update.hpp"
+#include "../adam/nadamax_update.hpp"
+#include "../adam/optimisticadam_update.hpp"
+#include "demon_adam_update.hpp"
+
+namespace ens {
+
+/**
+ * DemonAdam automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title  = {Decaying momentum helps neural network training},
+ *   author = {John Chen and Cameron Wolfe and Zhao Li
+ *             and Anastasios Kyrillidis},
+ *   url    = {https://arxiv.org/abs/1910.04952}
+ *   year   = {2019}
+ * }
+ *
+ * DemonAdam can optimize differentiable separable functions. For more details,
+ * see the documentation on function types include with this distribution or on
+ * the ensmallen website.
+ *
+ * @tparam UpdateRule Adam optimizer update rule to be used.
+ */
+template<typename UpdateRule = AdamUpdate>
+class DemonAdamType
+{
+ public:
+  /**
+   * Construct the DemonAdam optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param momentum The initial momentum coefficient.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+   *     estimates.
+   * @param eps Value used to initialise the mean squared gradient parameter.
+      * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *     final objective obtained on the last pass over the data).
+   */
+  DemonAdamType(const double stepSize = 0.001,
+                const size_t batchSize = 32,
+                const double momentum = 0.9,
+                const double beta1 = 0.9,
+                const double beta2 = 0.999,
+                const double eps = 1e-8,
+                const size_t maxIterations = 100000,
+                const double tolerance = 1e-5,
+                const bool shuffle = true,
+                const bool resetPolicy = true,
+                const bool exactObjective = false) :
+      optimizer(stepSize,
+                batchSize,
+                maxIterations,
+                tolerance,
+                shuffle,
+                DemonAdamUpdate<UpdateRule>(maxIterations * batchSize,
+                    momentum, UpdateRule(eps, beta1, beta2)),
+                NoDecay(),
+                resetPolicy,
+                exactObjective)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Optimize the given function using DemonAdam. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return optimizer.template Optimize<
+        SeparableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the moment coefficient.
+  double Momentum() const { return optimizer.UpdatePolicy().Momentum(); }
+  //! Modify the moment coefficient.
+  double& Momentum() { return optimizer.UpdatePolicy().Momentum(); }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations()
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with DemonAdam policy.
+  SGD<DemonAdamUpdate<UpdateRule>> optimizer;
+};
+
+using DemonAdam = DemonAdamType<AdamUpdate>;
+
+using DemonAdaMax = DemonAdamType<AdaMaxUpdate>;
+
+using DemonAMSGrad = DemonAdamType<AMSGradUpdate>;
+
+using DemonNadam = DemonAdamType<NadamUpdate>;
+
+using DemonNadaMax = DemonAdamType<NadaMaxUpdate>;
+
+using DemonOptimisticAdam = DemonAdamType<OptimisticAdamUpdate>;
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/demon_adam/demon_adam_update.hpp

@@ -0,0 +1,174 @@
+/**
+ * @file demon_sgd_update.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of DemonAdam.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_ADAM_DEMON_ADAM_UPDATE_HPP
+#define ENSMALLEN_DEMON_ADAM_DEMON_ADAM_UPDATE_HPP
+
+#include <assert.h>
+
+namespace ens {
+
+/**
+ * DemonAdam automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ * @endcode
+ *
+ * @tparam UpdateRule DemonAdam optimizer update rule to be used.
+ */
+template<typename UpdateRule>
+class DemonAdamUpdate
+{
+ public:
+  /**
+   * Construct the DemonAdam update policy with the given parameters.
+   *
+   * @param momentumIterations The number of iterations before the momentum
+   *     will decay to zero.
+   * @param momentum The initial momentum coefficient.
+   * @param adamUpdate Instantiated Adam update policy used to adjust the given
+   *     parameters.
+   */
+  DemonAdamUpdate(const size_t momentumIterations = 100,
+                  const double momentum = 0.9,
+                  const UpdateRule& adamUpdate = UpdateRule()) :
+      T(momentumIterations),
+      betaInit(momentum),
+      t(0),
+      adamUpdateInst(adamUpdate)
+  {
+    // Make sure the momentum iterations parameter is non-zero.
+    assert(momentumIterations != 0 && "The number of iterations before the "
+        "momentum will decay is zero, make sure the max iterations and "
+        "batch size parameter is set correctly. "
+        "Default: momentumIterations = maxIterations / batchSize.");
+  }
+
+  //! Get the momentum coefficient.
+  double Momentum() const { return betaInit; }
+  //! Modify the momentum coefficient.
+  double& Momentum() { return betaInit; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return t; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return t; }
+
+  //! Get the momentum ion number.
+  size_t MomentumIterations() const { return T; }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations() { return T; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    // Convenient typedef.
+    typedef typename UpdateRule::template Policy<MatType, GradType>
+        InstUpdateRuleType;
+    typedef typename MatType::elem_type ElemType;
+
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated PadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(DemonAdamUpdate& parent,
+           const size_t rows,
+           const size_t cols) :
+      parent(parent),
+      adamUpdate(new InstUpdateRuleType(parent.adamUpdateInst, rows, cols)),
+      betaInit(ElemType(parent.betaInit))
+    { /* Nothing to do here */ }
+
+    /**
+     * Clean any memory associated with the Polciy object.
+     */
+    ~Policy()
+    {
+      delete adamUpdate;
+    }
+
+    /**
+     * Update step for DamonAdam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      ElemType decayRate = 1;
+      if (parent.t > 0)
+        decayRate = 1 - ElemType((double) parent.t / (double) parent.T);
+
+      const ElemType betaDecay = betaInit * decayRate;
+      const ElemType beta = betaDecay / ((1 - betaInit) + betaDecay);
+
+      // Perform the update.
+      iterate *= beta;
+
+      // Apply the adam update.
+      adamUpdate->Update(iterate, stepSize, gradient);
+
+      // Increment the iteration counter variable.
+      ++parent.t;
+    }
+
+   private:
+    // Instantiated parent object.
+    DemonAdamUpdate<UpdateRule>& parent;
+
+    // The update policy.
+    InstUpdateRuleType* adamUpdate;
+
+    // Optimizer parameter converted to the element type of the optimization.
+    ElemType betaInit;
+  };
+
+ private:
+  //! The number of momentum iterations.
+  size_t T;
+
+  //! Initial momentum coefficient.
+  double betaInit;
+
+  //! The number of iterations.
+  size_t t;
+
+  //! The adam update policy.
+  UpdateRule adamUpdateInst;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/demon_sgd/demon_sgd.hpp

@@ -0,0 +1,178 @@
+/**
+ * @file demon_sgd.hpp
+ * @author Marcus Edel
+ *
+ * Definition of DemonSGD.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_SGD_DEMON_SGD_HPP
+#define ENSMALLEN_DEMON_SGD_DEMON_SGD_HPP
+
+#include "../sgd/sgd.hpp"
+#include "demon_sgd_update.hpp"
+
+namespace ens {
+
+/**
+ * DemonSGD automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title  = {Decaying momentum helps neural network training},
+ *   author = {John Chen and Cameron Wolfe and Zhao Li
+ *             and Anastasios Kyrillidis},
+ *   url    = {https://arxiv.org/abs/1910.04952}
+ *   year   = {2019}
+ * }
+ *
+ * DemonSGD can optimize differentiable separable functions. For more details,
+ * see the documentation on function types include with this distribution or on
+ * the ensmallen website.
+ */
+class DemonSGD
+{
+ public:
+  /**
+   * Construct the DemonSGD optimizer with the given function and parameters.
+   * The defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param momentum The initial momentum coefficient.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *     final objective obtained on the last pass over the data).
+   */
+  DemonSGD(const double stepSize = 0.001,
+           const size_t batchSize = 32,
+           const double momentum = 0.9,
+           const size_t maxIterations = 100000,
+           const double tolerance = 1e-5,
+           const bool shuffle = true,
+           const bool resetPolicy = true,
+           const bool exactObjective = false) :
+      optimizer(stepSize,
+                batchSize,
+                maxIterations,
+                tolerance,
+                shuffle,
+                DemonSGDUpdate(maxIterations * batchSize, momentum),
+                NoDecay(),
+                resetPolicy,
+                exactObjective)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Optimize the given function using DemonSGD. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return optimizer.template Optimize<
+        SeparableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the moment coefficient.
+  double Momentum() const { return optimizer.UpdatePolicy().Momentum(); }
+  //! Modify the moment coefficient.
+  double& Momentum() { return optimizer.UpdatePolicy().Momentum(); }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations()
+  { return optimizer.UpdatePolicy().MomentumIterations(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with DemonSGD policy.
+  SGD<DemonSGDUpdate> optimizer;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/demon_sgd/demon_sgd_update.hpp

@@ -0,0 +1,145 @@
+/**
+ * @file demon_sgd_update.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of DemonSGD.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DEMON_SGD_DEMON_SGD_UPDATE_HPP
+#define ENSMALLEN_DEMON_SGD_DEMON_SGD_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * DemonSGD automatically decays momentum, motivated by decaying the total
+ * contribution of a gradient to all future updates.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @misc{
+ *   title   = {Decaying momentum helps neural network training},
+ *   author  = {John Chen and Cameron Wolfe and Zhao Li
+ *              and Anastasios Kyrillidis},
+ *   url     = {https://arxiv.org/abs/1910.04952}
+ *   year    = {2019}
+ * }
+ * @endcode
+ */
+class DemonSGDUpdate
+{
+ public:
+  /**
+   * Construct the DemonSGD update policy with the given parameters.
+   *
+   * @param momentumIterations The number of iterations before the momentum
+   *     will decay to zero.
+   * @param momentum The initial momentum coefficient.
+   */
+  DemonSGDUpdate(const size_t momentumIterations = 100,
+                 const double momentum = 0.9) :
+      T(momentumIterations),
+      betaInit(momentum),
+      t(0)
+  {
+    // Make sure the momentum iterations parameter is non-zero.
+    assert(momentumIterations != 0 && "The number of iterations before the "
+        "momentum will decay is zero, make sure the max iterations and "
+        "batch size parameter is set correctly. "
+        "Default: momentumIterations = maxIterations * batchSize.");
+  }
+
+  //! Get the momentum coefficient.
+  double Momentum() const { return betaInit; }
+  //! Modify the momentum coefficient.
+  double& Momentum() { return betaInit; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return t; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return t; }
+
+  //! Get the momentum iteration number.
+  size_t MomentumIterations() const { return T; }
+  //! Modify the momentum iteration number.
+  size_t& MomentumIterations() { return T; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    typedef typename MatType::elem_type ElemType;
+
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated PadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(DemonSGDUpdate& parent,
+           const size_t /* rows */,
+           const size_t /* cols */) :
+      parent(parent),
+      betaInit(ElemType(parent.betaInit))
+    { /* Nothing to do here */ }
+
+    /**
+     * Update step for DemonSGD.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      ElemType decayRate = 1;
+      if (parent.t > 0)
+        decayRate = 1 - ElemType((double) parent.t / (double) parent.T);
+
+      const ElemType betaDecay = betaInit * decayRate;
+      const ElemType beta = betaDecay / ((1 - betaInit) + betaDecay);
+
+      // Perform the update.
+      iterate *= beta;
+      iterate -= ElemType(stepSize) * gradient;
+
+      // Increment the iteration counter variable.
+      ++parent.t;
+    }
+
+   private:
+    // Instantiated parent object.
+    DemonSGDUpdate& parent;
+
+    // Optimizer parameter converted to the element type of the optimization.
+    ElemType betaInit;
+  };
+
+ private:
+  // The number of momentum iterations.
+  size_t T;
+
+  // Initial momentum coefficient.
+  double betaInit;
+
+  // The number of iterations.
+  size_t t;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ens_version.hpp

@@ -12,20 +12,20 @@
 
 // This follows the Semantic Versioning pattern defined in https://semver.org/.
 
-#define ENS_VERSION_MAJOR 2
+#define ENS_VERSION_MAJOR 3
 // The minor version is two digits so regular numerical comparisons of versions
 // work right.  The first minor version of a release is always 10.
-#define ENS_VERSION_MINOR 16
-#define ENS_VERSION_PATCH 2
+#define ENS_VERSION_MINOR 11
+#define ENS_VERSION_PATCH 0
 // If this is a release candidate, it will be reflected in the version name
 // (i.e. the version name will be "RC1", "RC2", etc.).  Otherwise the version
 // name will typically be a seemingly arbitrary set of words that does not
 // contain the capitalized string "RC".
-#define ENS_VERSION_NAME "Severely Dented Can Of Polyurethane"
+#define ENS_VERSION_NAME "Sunny Day"
 // Incorporate the date the version was released.
-#define ENS_VERSION_YEAR "2021"
-#define ENS_VERSION_MONTH "03"
-#define ENS_VERSION_DAY "24"
+#define ENS_VERSION_YEAR "2025"
+#define ENS_VERSION_MONTH "12"
+#define ENS_VERSION_DAY "15"
 
 namespace ens {
 

include/ensmallen_bits/eve/eve.hpp

@@ -106,7 +106,7 @@ class Eve
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/eve/eve_impl.hpp

@@ -49,8 +49,8 @@ template<typename SeparableFunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 Eve::Optimize(SeparableFunctionType& function,
               MatType& iterateIn,
               CallbackTypes&&... callbacks)
@@ -98,7 +98,7 @@ Eve::Optimize(SeparableFunctionType& function,
   v.zeros();
 
   // Now iterate!
-  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
   BaseGradType gradient(iterate.n_rows, iterate.n_cols);
   const size_t actualMaxIterations = (maxIterations == 0) ?
       std::numeric_limits<size_t>::max() : maxIterations;
@@ -123,30 +123,40 @@ Eve::Optimize(SeparableFunctionType& function,
 
     terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
         objective, gradient, callbacks...);
+    if (terminate)
+      break;
 
-    m *= beta1;
-    m += (1 - beta1) * gradient;
+    m *= ElemType(beta1);
+    m += (1 - ElemType(beta1)) * gradient;
 
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
+    v *= ElemType(beta2);
+    v += (1 - ElemType(beta2)) * (gradient % gradient);
 
-    const double biasCorrection1 = 1.0 - std::pow(beta1, (double) (i + 1));
-    const double biasCorrection2 = 1.0 - std::pow(beta2, (double) (i + 1));
+    const ElemType biasCorrection1 =
+        1 - std::pow(ElemType(beta1), ElemType(i + 1));
+    const ElemType biasCorrection2 =
+        1 - std::pow(ElemType(beta2), ElemType(i + 1));
 
     if (i > 0)
     {
       const ElemType d = std::abs(objective - lastObjective) /
-          (std::min(objective, lastObjective) + epsilon);
+          (std::min(objective, lastObjective) + ElemType(epsilon));
 
-      dt *= beta3;
-      dt += (1 - beta3) * std::min(std::max(d, ElemType(1.0 / clip)),
+      dt *= ElemType(beta3);
+      dt += (1 - ElemType(beta3)) * std::min(std::max(d, ElemType(1.0 / clip)),
           ElemType(clip));
     }
 
     lastObjective = objective;
 
-    iterate -= stepSize / dt * (m / biasCorrection1) /
-        (arma::sqrt(v / biasCorrection2) + epsilon);
+    // TODO: remove in ensmallen 4.0.0.
+    #if defined(ENS_OLD_SEPARABLE_STEP_BEHAVIOR)
+    iterate -= ElemType(stepSize) / dt * (m / biasCorrection1) /
+        (sqrt(v / biasCorrection2) + ElemType(epsilon));
+    #else
+    iterate -= (ElemType(stepSize) / (dt * effectiveBatchSize)) *
+        (m / biasCorrection1) / (sqrt(v / biasCorrection2) + ElemType(epsilon));
+    #endif
 
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
 
@@ -172,9 +182,7 @@ Eve::Optimize(SeparableFunctionType& function,
         return overallObjective;
       }
 
-      if (std::abs(lastOverallObjective - overallObjective) < tolerance ||
-          Callback::BeginEpoch(*this, f, iterate, epoch, overallObjective,
-              callbacks...))
+      if (std::abs(lastOverallObjective - overallObjective) < tolerance)
       {
         Info << "Eve: minimized within tolerance " << tolerance << "; "
             << "terminating optimization." << std::endl;
@@ -183,6 +191,9 @@ Eve::Optimize(SeparableFunctionType& function,
         return overallObjective;
       }
 
+      terminate |= Callback::BeginEpoch(*this, f, iterate, epoch,
+          overallObjective, callbacks...);
+
       // Reset the counter variables.
       lastOverallObjective = overallObjective;
       overallObjective = 0;
@@ -206,7 +217,9 @@ Eve::Optimize(SeparableFunctionType& function,
       const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
       overallObjective += objective;
 
-      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+      // The optimization is finished, so we don't need to care about the result
+      // of the callback.
+      (void) Callback::Evaluate(*this, f, iterate, objective, callbacks...);
     }
   }
 

include/ensmallen_bits/fasta/fasta.hpp

@@ -0,0 +1,220 @@
+/**
+ * @file fasta.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of FASTA (Fast Adaptive Shrinkage/Thresholding Algorithm).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FASTA_FASTA_HPP
+#define ENSMALLEN_FASTA_FASTA_HPP
+
+#include "../fbs/l1_penalty.hpp"
+#include "../fbs/l1_constraint.hpp"
+
+namespace ens {
+
+/**
+ * FASTA (Fast Adaptive Shrinkage/Thresholding Algorithm) is a proximal
+ * gradient optimization technique for optimizing a function of the form
+ *
+ *   h(x) = f(x) + g(x)
+ *
+ * where f(x) is a differentiable function and g(x) is an arbitrary
+ * non-differentiable function.  In such a situation, standard gradient descent
+ * techniques cannot work because of the non-differentiability of g(x).  To work
+ * around this, FASTA takes a _forward step_ that is just a gradient descent
+ * step on f(x), and then a _backward step_ that is the _proximal operator_
+ * corresponding to g(x).  This continues until convergence.
+ *
+ * This implementation of FASTA allows specification of the backward step (or
+ * proximal operator) via the `BackwardStepType` template parameter.  When using
+ * FBS, the differentiable `FunctionType` given to `Optimize()` should be f(x),
+ * *not* the combined function h(x).  g(x) should be specified by the choice of
+ * `BackwardStepType` (e.g. `L1Penalty` or `L1Maximum`).  The `Optimize()`
+ * function will then return optimized coordinates for h(x), not f(x).
+ *
+ * For more information, see the following paper:
+ *
+ * ```
+ * @article{goldstein2014field,
+ *   title={A field guide to forward-backward splitting with a FASTA
+ *       implementation},
+ *   author={Goldstein, Tom and Studer, Christoph and Baraniuk, Richard},
+ *   journal={arXiv preprint arXiv:1411.3406},
+ *   year={2014}
+ * }
+ * ```
+ */
+template<typename BackwardStepType = L1Penalty>
+class FASTA
+{
+ public:
+  /**
+   * Construct the FASTA optimizer with the given options, using a
+   * default-constructed BackwardStepType.
+   */
+  FASTA(const size_t maxIterations = 10000,
+        const double tolerance = 1e-7,
+        const size_t maxLineSearchSteps = 50,
+        const double stepSizeAdjustment = 2.0,
+        const size_t lineSearchLookback = 10,
+        const bool estimateStepSize = true,
+        const size_t estimateTrials = 10,
+        const double maxStepSize = 0.001);
+
+  /**
+   * Construct the FASTA optimizer with the given options.
+   */
+  FASTA(BackwardStepType backwardStepType,
+        const size_t maxIterations = 10000,
+        const double tolerance = 1e-7,
+        const size_t maxLineSearchSteps = 50,
+        const double stepSizeAdjustment = 2.0,
+        const size_t lineSearchLookback = 10,
+        const bool estimateStepSize = true,
+        const size_t estimateTrials = 10,
+        const double maxStepSize = 0.001);
+
+  /**
+   * Optimize the given function using FASTA.  The given starting
+   * point will be modified to store the finishing point of the algorithm,
+   * the final objective value is returned.
+   *
+   * The FunctionType template class must provide the following functions:
+   *
+   *   double Evaluate(const arma::mat& coordinates);
+   *   void Gradient(const arma::mat& coordinates,
+   *                 arma::mat& gradient);
+   *
+   * @tparam FunctionType Type of function to be optimized.
+   * @tparam MatType Type of objective matrix.
+   * @tparam GradType Type of gradient matrix (default is MatType).
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to be optimized.
+   * @param iterate Input with starting point, and will be modified to save
+   *                the output optimial solution coordinates.
+   * @param callbacks Callback functions.
+   * @return Objective value at the final solution.
+   */
+  template<typename FunctionType, typename MatType, typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the backward step object.
+  const BackwardStepType& BackwardStep() const { return backwardStep; }
+  //! Modify the backward step object.
+  BackwardStepType& BackwardStep() { return backwardStep; }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return maxIterations; }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return maxIterations; }
+
+  //! Get the tolerance on the gradient norm for termination.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance on the gradient norm for termination.
+  double& Tolerance() { return tolerance; }
+
+  //! Get the maximum number of line search steps.
+  size_t MaxLineSearchSteps() const { return maxLineSearchSteps; }
+  //! Modify the maximum number of line search steps.
+  size_t& MaxLineSearchSteps() { return maxLineSearchSteps; }
+
+  //! Get the step size adjustment parameter.
+  double StepSizeAdjustment() const { return stepSizeAdjustment; }
+  //! Modify the step size adjustment parameter.
+  double& StepSizeAdjustment() { return stepSizeAdjustment; }
+
+  //! Get the maximum number of iterations to look back during a line search.
+  size_t LineSearchLookback() const { return lineSearchLookback; }
+  //! Modify the maximum number of iterations to look back during a line search.
+  size_t& LineSearchLookback() { return lineSearchLookback; }
+
+  //! Get whether or not to estimate the initial step size.
+  bool EstimateStepSize() const { return estimateStepSize; }
+  //! Modify whether or not to estimate the initial step size.
+  bool& EstimateStepSize() { return estimateStepSize; }
+
+  //! Get the number of trials to use for Lipschitz constant estimation.
+  size_t EstimateTrials() const { return estimateTrials; }
+  //! Modify the number of trials to use for Lipschitz constant estimation.
+  size_t& EstimateTrials() { return estimateTrials; }
+
+  //! Get the maximum step size.  If Optimize() has been called, this will
+  //! contain the estimated maximum step size value.
+  double MaxStepSize() const { return maxStepSize; }
+  //! Modify the step size (ignored if EstimateStepSize() is true).
+  double& MaxStepSize() { return maxStepSize; }
+
+ private:
+  //! Utility function: fill with random values.
+  template<typename MatType>
+  static void RandomFill(MatType& x,
+                         const size_t rows,
+                         const size_t cols,
+                         const typename MatType::elem_type maxVal);
+
+  template<typename eT>
+  static void RandomFill(arma::SpMat<eT>& x,
+                         const size_t rows,
+                         const size_t cols,
+                         const eT maxVal);
+
+  template<typename FunctionType, typename MatType>
+  void EstimateLipschitzStepSize(FunctionType& f, const MatType& x);
+
+  //! The instantiated backward step object.
+  BackwardStepType backwardStep;
+
+  //! The maximum number of allowed iterations.
+  size_t maxIterations;
+
+  //! The tolerance for termination.
+  double tolerance;
+
+  //! The maximum number of line search trials.
+  size_t maxLineSearchSteps;
+
+  //! The step size adjustment parameter for the line search.
+  double stepSizeAdjustment;
+
+  //! The maximum number of iterations to look back during a line search.
+  size_t lineSearchLookback;
+
+  //! Whether or not to try and estimate the initial step size.
+  bool estimateStepSize;
+
+  //! Number of trials to use for initial step size estimation.
+  size_t estimateTrials;
+
+  //! The maximum step size to use (estimated if estimateStepSize is true).
+  double maxStepSize;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "fasta_impl.hpp"
+
+#endif

include/ensmallen_bits/fasta/fasta_impl.hpp

@@ -0,0 +1,549 @@
+/**
+ * @file fasta_impl.hpp
+ * @author Ryan Curtin
+ *
+ * Implementation of FASTA (Fast Adaptive Shrinkage/Thresholding Algorithm).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FASTA_FASTA_IMPL_HPP
+#define ENSMALLEN_FASTA_FASTA_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "fasta.hpp"
+
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+//! Constructor of the FBS class.
+template<typename BackwardStepType>
+FASTA<BackwardStepType>::FASTA(const size_t maxIterations,
+                               const double tolerance,
+                               const size_t maxLineSearchSteps,
+                               const double stepSizeAdjustment,
+                               const size_t lineSearchLookback,
+                               const bool estimateStepSize,
+                               const size_t estimateTrials,
+                               const double maxStepSize) :
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    maxLineSearchSteps(maxLineSearchSteps),
+    stepSizeAdjustment(stepSizeAdjustment),
+    lineSearchLookback(lineSearchLookback),
+    estimateStepSize(estimateStepSize),
+    estimateTrials(estimateTrials),
+    maxStepSize(maxStepSize)
+{
+  // Check estimateSteps parameter.
+  if (estimateStepSize && estimateTrials == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): estimateTrials must be greater"
+        " than 0!");
+  }
+
+  if (lineSearchLookback == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): lineSearchLookback cannot be "
+        "0!");
+  }
+}
+
+template<typename BackwardStepType>
+FASTA<BackwardStepType>::FASTA(BackwardStepType backwardStep,
+                               const size_t maxIterations,
+                               const double tolerance,
+                               const size_t maxLineSearchSteps,
+                               const double stepSizeAdjustment,
+                               const size_t lineSearchLookback,
+                               const bool estimateStepSize,
+                               const size_t estimateTrials,
+                               const double maxStepSize) :
+    backwardStep(std::move(backwardStep)),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    maxLineSearchSteps(maxLineSearchSteps),
+    stepSizeAdjustment(stepSizeAdjustment),
+    lineSearchLookback(lineSearchLookback),
+    estimateStepSize(estimateStepSize),
+    estimateTrials(estimateTrials),
+    maxStepSize(maxStepSize)
+{
+  // Check estimateSteps parameter.
+  if (estimateStepSize && estimateTrials == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): estimateTrials must be greater"
+        " than 0!");
+  }
+
+  if (lineSearchLookback == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): lineSearchLookback cannot be "
+        "0!");
+  }
+}
+
+//! Optimize the function (minimize).
+template<typename BackwardStepType>
+template<typename FunctionType, typename MatType, typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
+FASTA<BackwardStepType>::Optimize(FunctionType& function,
+                                  MatType& iterateIn,
+                                  CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
+  FullFunctionType& f = static_cast<FullFunctionType&>(function);
+
+  // Make sure we have all necessary functions.
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  // Sanity check: make sure lineSearchLookback is valid.
+  if (lineSearchLookback == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): lineSearchLookback cannot be "
+        "0!");
+  }
+
+  // Here we make a copy because we will use std::move() internally, and if
+  // iterateIn is an alias, this is unsafe.  We will copy the final result back
+  // to iterateIn at the end.
+  BaseMatType x(iterateIn);
+
+  // To keep track of the function value.
+  ElemType currentFObj = f.Evaluate(x);
+  ElemType currentGObj = backwardStep.Evaluate(x);
+  ElemType currentObj = currentFObj + currentGObj;
+
+  // This will be the denominator of the normalized residual termination
+  // condition.
+  ElemType firstResidual = ElemType(0);
+
+  // This will be used in the non-monotone line search, to track the last
+  // several function values.
+  arma::Col<ElemType> lastFObjs(lineSearchLookback);
+  lastFObjs.fill(std::numeric_limits<ElemType>::min());
+  size_t currentObjPos = 0;
+
+  BaseGradType g(x.n_rows, x.n_cols);
+  BaseMatType lastXHat; // Used for residual checks.
+  BaseMatType lastX; // Used for residual and alpha reset checks.
+  BaseMatType xHat; // Used for residual checks.
+  BaseMatType lpaX = x; // Used for alpha reset check.
+  ElemType alpha = ElemType(1); // Initialize alpha^1 = 1.
+  ElemType lastAlpha = alpha;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // First, estimate the Lipschitz constant to set the initial/maximum step
+  // size, if the user asked us to.
+  if (estimateStepSize)
+    EstimateLipschitzStepSize(f, x);
+
+  // Keep track of the last step size we used.
+  ElemType currentStepSize = (ElemType) maxStepSize;
+  ElemType lastStepSize = (ElemType) maxStepSize;
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
+  Callback::BeginOptimization(*this, f, x, callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
+  {
+    // During this optimization, we want to optimize h(x) = f(x) + g(x).
+    // f(x) is `f`, but g(x) is specified by `BackwardStepType`.
+
+    // The first step is to compute a step size via a non-monotone line search.
+    // To do this, we need to compute the gradient f'(y) as required by the line
+    // search condition in Eq. (38).  Note that our code does a little sleight
+    // of hand, and so `x` stores what the paper calls `y^k` here.  (See the
+    // code for the adaptive step below.)
+    currentFObj = f.EvaluateWithGradient(x, g);
+    terminate |= Callback::EvaluateWithGradient(*this, f, x, currentFObj, g,
+        callbacks...);
+
+    // Use backtracking non-monotone line search to find the best step size.
+    // This is the version from the FASTA paper, but with a minor modification:
+    // we start our search at the last step size, and allow the search to
+    // increase the step size up to the maximum step size if it can.  This is a
+    // more effective heuristic than simply starting at the largest allowable
+    // step size and shrinking from there, especially in regions where the
+    // gradient norm is small.  It is also more effective than simply starting
+    // at the last step size and shrinking from there, as it prevents getting
+    // "stuck" with a very small step size.
+    bool lsDone = false;
+    size_t lsTrial = 0;
+    bool increasing = false; // Will be set during the first iteration.
+    ElemType lastFObj = ElemType(0);
+    BaseMatType lsLastX; // Only used in increasing mode.
+    BaseMatType lsLastXHat; // Only used in increasing mode.
+    BaseMatType xDiff;
+
+    lastX = std::move(x);
+    lastStepSize = currentStepSize;
+    currentStepSize = std::min(currentStepSize, (ElemType) maxStepSize);
+
+    // Ensure that the last `lineSearchLookback` objective values are recorded
+    // properly.
+    lastFObjs[currentObjPos] = currentFObj;
+    currentObjPos = (currentObjPos + 1) % lineSearchLookback;
+    const ElemType strictMaxFObj = currentFObj;
+    const ElemType maxFObj = lastFObjs.max();
+
+    while (!lsDone && !terminate)
+    {
+      if (lsTrial == maxLineSearchSteps)
+      {
+        if (increasing)
+        {
+          Warn << "FASTA::Optimize(): line search reached maximum number of "
+              << "steps (" << maxLineSearchSteps << "); using step size "
+              << currentStepSize << "." << std::endl;
+          break; // The step size is still valid.
+        }
+        else
+        {
+          Warn << "FASTA::Optimize(): could not find valid step size in range "
+              << "(0, " << maxStepSize << "]!  Terminating optimization."
+              << std::endl;
+          terminate = true;
+          break;
+        }
+      }
+
+      // If the step size has converged to zero, we are done.
+      if (currentStepSize == ElemType(0))
+      {
+        Warn << "FASTA::Optimize(): computed zero step size; terminating "
+            << "optimization." << std::endl;
+        terminate = true;
+        break;
+      }
+
+      // Perform forward update into x.
+      xHat = lastX - currentStepSize * g;
+      // (We must store xHat separately for the residual, so this copy is
+      // necessary.)
+      x = xHat;
+      backwardStep.ProximalStep(x, currentStepSize);
+
+      // Compute objective of new point.
+      const ElemType fObj = f.Evaluate(x);
+      terminate |= Callback::Evaluate(*this, f, x, fObj, callbacks...);
+
+      // Compute the quadratic approximation of the objective (the condition in
+      // Eq. (38)).
+      xDiff = (x - lastX);
+
+      // Note: since we allow the step size to increase, we have to modify the
+      // non-monotone line search a little bit to keep things from diverging.
+      // Specifically, if we are increasing the step size, then we force a
+      // monotone line search (by looking only at the previous function value).
+      // It is only when we are decreasing the step size that we allow
+      // relaxation.
+      const ElemType relaxedCond = maxFObj + dot(xDiff, g) +
+          (1 / (2 * currentStepSize)) * dot(xDiff, xDiff);
+      const ElemType strictCond = strictMaxFObj + dot(xDiff, g) +
+          (1 / (2 * currentStepSize)) * dot(xDiff, xDiff);
+
+      // If we're on the first iteration, we don't know if we should be
+      // searching for a step size by increasing or decreasing the step size.
+      // (Remember that our valid ranges of step sizes are [0, maxStepSize], and
+      // we are starting at lastStepSize.)
+      //
+      // Thus, if the condition is satisfied, let's try increasing the step size
+      // until it's no longer satisfied.  Otherwise, we will have to decrease
+      // the step size.
+      if (lsTrial == 0)
+      {
+        increasing = ((fObj <= strictCond) && (std::isfinite(fObj)));
+      }
+
+      if (increasing)
+      {
+        // If we are in "increasing" mode, then termination occurs on the first
+        // iteration when the strict condition is *not* satisfied (and we use
+        // the last step size).
+        if ((fObj > strictCond) || (!std::isfinite(fObj)))
+        {
+          lsDone = true;
+          x = std::move(lsLastX);
+          xHat = std::move(lsLastXHat);
+          currentFObj = lastFObj;
+          currentStepSize = lastStepSize; // Take one step backwards.
+        }
+        else if (currentStepSize == (ElemType) maxStepSize)
+        {
+          // The condition is still satisfied, but the step size will be too big
+          // if we take another step.  Go back to the maximum step size.
+          lsDone = true;
+          currentFObj = fObj;
+        }
+        else
+        {
+          // The condition is still satisfied; increase the step size.
+          lastStepSize = currentStepSize;
+          currentStepSize *= ElemType(stepSizeAdjustment);
+          lsLastX = std::move(x);
+          lsLastXHat = std::move(xHat);
+          lastFObj = fObj;
+          ++lsTrial;
+        }
+      }
+      else
+      {
+        // If we are in "decreasing" mode, then termination occurs on the first
+        // iteration when the relaxed condition is satisfied.
+        if ((fObj <= relaxedCond) && (std::isfinite(fObj)))
+        {
+          lsDone = true;
+          currentFObj = fObj;
+        }
+        else
+        {
+          // The condition is not yet satisfied; decrease the step size.
+          currentStepSize /= ElemType(stepSizeAdjustment);
+          ++lsTrial;
+        }
+      }
+    }
+
+    if (!lsDone)
+    {
+      // The line search failed, so terminate.
+      Warn << "FASTA::Optimize(): non-monotone line search failed after "
+          << maxLineSearchSteps << " steps; terminating optimization."
+          << std::endl;
+      x = std::move(lastX);
+      terminate = true;
+    }
+
+    // If we terminated during the line search, we are done.
+    if (terminate)
+      break;
+
+    // Now that we have taken a step, compute the full objective by computing
+    // g(x).
+    currentGObj = backwardStep.Evaluate(x);
+    currentObj = currentFObj + currentGObj;
+
+    // Output current objective function.
+    Info << "FASTA::Optimize(): iteration " << i << ", combined objective "
+        << currentObj << " (f(x) = " << currentFObj << ", g(x) = "
+        << currentGObj << "), step size " << currentStepSize << "."
+        << std::endl;
+
+    // Sanity check for divergence.
+    if ((i > 1) && !std::isfinite(currentObj))
+    {
+      Warn << "FASTA::Optimize(): objective diverged to "
+          << currentObj << "; terminating optimization." << std::endl;
+      terminate = true;
+      break;
+    }
+
+    // Now, check for convergence.  The FASTA convergence check uses both the
+    // normalized residual and the relative residual, stopping when either
+    // becomes sufficiently small.  The check depends on x before and after the
+    // proximal step.
+
+    // Compute residual.  This is Eq. (40) in the paper.
+    const ElemType residual = norm(g + (xHat - x) / currentStepSize, 2);
+
+    // If this is the first iteration, store the residual as the first residual.
+    if (i == 1)
+      firstResidual = residual;
+
+    // First, check the normalized residual for convergence.  This is Eq. (43)
+    // in the paper.
+    const ElemType eps = 20 * std::numeric_limits<ElemType>::epsilon();
+    const ElemType normalizedResidual = residual / (firstResidual + eps);
+
+    if ((i < 10) && (normalizedResidual < ElemType(1e-5)))
+    {
+      // Heuristic: sometimes the optimization starts in such an awful place
+      // that we are able to make huge amounts of progress in the first few
+      // iterations.  In this case, reset the firstResidual to the slightly
+      // better point we get to by the tenth iterate.
+      firstResidual = residual;
+    }
+    else if ((i > 10) && (normalizedResidual < tolerance))
+    {
+      Info << "FASTA::Optimize(): normalized residual minimized within "
+          << "tolerance " << tolerance << "; terminating optimization."
+          << std::endl;
+      break;
+    }
+
+    // Next, check the relative residual for convergence.  This is Eq. (42) in
+    // the paper.
+    const ElemType gNorm = norm(g, 2);
+    const ElemType proxStepNorm = norm((xHat - x) / currentStepSize, 2);
+
+    const ElemType relativeResidual = residual /
+        (std::max(gNorm, proxStepNorm) + 20 * eps);
+
+    if (relativeResidual < tolerance)
+    {
+      Info << "FASTA::Optimize(): relative residual minimized within "
+          << "tolerance " << tolerance << "; terminating optimization."
+          << std::endl;
+      break;
+    }
+
+    // Compute updated prediction parameter alpha.
+    lastAlpha = alpha;
+    alpha = (1 + std::sqrt(1 + 4 * std::pow(alpha, ElemType(2)))) / 2;
+
+    // Take a predictive step.
+    BaseMatType y = x + ((lastAlpha - 1) / alpha) * (x - lpaX);
+
+    // Sometimes alpha can get to be too large; this restart scheme is taken
+    // originally from O'Donoghue and Candes, "Adaptive restart for accelerated
+    // gradient schemes", 2012.
+    //
+    // The notation is confusing here when compared with Eq. (37) in the paper.
+    // This is because the paper is poorly notated, although it's not clear much
+    // has been done here to improve things.  To translate:
+    //
+    //    Paper   Code    Explanation
+    //
+    //    y^k     lastX   This is the result of the predictive step on the
+    //                    previous iteration.  In our code, we apply the
+    //                    predictive step to x, which next iteration becomes
+    //                    lastX.
+    //
+    //    x^k     x       This is the iterate before the predictive step, this
+    //                    iteration.
+    //
+    //    x^k-1   lpaX    "Last Pre-Accelerated X"---we have to take a specific
+    //                    step to store this.
+    //
+    const ElemType restartCheck = dot(lastX - x, x - lpaX);
+    if (restartCheck > 0)
+    {
+      Info << "FASTA::Optimize(): alpha too large (" << alpha << "); reset to "
+          << "1." << std::endl;
+      alpha = ElemType(1);
+      lastAlpha = ElemType(1);
+    }
+
+    lpaX = std::move(x);
+    x = std::move(y);
+
+    terminate |= Callback::StepTaken(*this, f, x, callbacks...);
+  }
+
+  if (!terminate)
+  {
+    Info << "FASTA::Optimize(): maximum iterations (" << maxIterations
+        << ") reached; terminating optimization." << std::endl;
+  }
+
+  Callback::EndOptimization(*this, f, x, callbacks...);
+
+  ((BaseMatType&) iterateIn) = x;
+  return currentObj;
+} // Optimize()
+
+template<typename BackwardStepType>
+template<typename MatType>
+void FASTA<BackwardStepType>::RandomFill(
+    MatType& x,
+    const size_t rows,
+    const size_t cols,
+    const typename MatType::elem_type maxVal)
+{
+  x.randu(rows, cols);
+  x *= maxVal;
+}
+
+template<typename BackwardStepType>
+template<typename eT>
+void FASTA<BackwardStepType>::RandomFill(
+    arma::SpMat<eT>& x,
+    const size_t rows,
+    const size_t cols,
+    const eT maxVal)
+{
+  eT density = eT(0.1);
+  // Try and keep the matrix from having too many elements.
+  if (rows * cols > 100000)
+    density = eT(0.01);
+  else if (rows * cols > 1000000)
+    density = eT(0.001);
+  else if (rows * cols > 10000000)
+    density = eT(0.0001);
+
+  x.sprandu(rows, cols, density);
+
+  // Make sure we got at least some nonzero elements...
+  while (x.n_nonzero == 0)
+  {
+    if (x.n_elem < 10)
+      x.sprandu(rows, cols, 1.0);
+    else
+      x.sprandu(rows, cols, 0.5);
+  }
+
+  x *= maxVal;
+}
+
+template<typename BackwardStepType>
+template<typename FunctionType, typename MatType>
+void FASTA<BackwardStepType>::EstimateLipschitzStepSize(
+    FunctionType& f,
+    const MatType& x)
+{
+  typedef typename MatType::elem_type ElemType;
+
+  // Sanity check for estimateSteps parameter.
+  if (estimateTrials == 0)
+  {
+    throw std::invalid_argument("FASTA::Optimize(): estimateTrials must be "
+        "greater than 0!");
+  }
+
+  const ElemType xMax = std::max(ElemType(1), 2 * x.max());
+  ElemType sum = ElemType(0);
+  MatType x1, x2, gx1, gx2;
+
+  for (size_t t = 0; t < estimateTrials; ++t)
+  {
+    RandomFill(x1, x.n_rows, x.n_cols, xMax);
+    RandomFill(x2, x.n_rows, x.n_cols, xMax);
+
+    f.Gradient(x1, gx1);
+    f.Gradient(x2, gx2);
+
+    // Compute a Lipschitz constant estimate.
+    const ElemType lEst = norm(gx1 - gx2, 2) / norm(x1 - x2, 2);
+    sum += lEst;
+  }
+
+  sum /= estimateTrials;
+  if (sum == 0)
+    maxStepSize = std::numeric_limits<ElemType>::max();
+  else
+    maxStepSize = (10 / sum);
+
+  Info << "FASTA::Optimize(): estimated a maximum step size of "
+      << maxStepSize << "." << std::endl;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/fbs/fbs.hpp

@@ -0,0 +1,153 @@
+/**
+ * @file fbs.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of Forward-Backward Splitting (FBS).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_FBS_HPP
+#define ENSMALLEN_FBS_FBS_HPP
+
+#include "l1_penalty.hpp"
+#include "l1_constraint.hpp"
+
+namespace ens {
+
+/**
+ * Forward-Backward Splitting is a proximal gradient optimization technique for
+ * optimizing a function of the form
+ *
+ *   h(x) = f(x) + g(x)
+ *
+ * where f(x) is a differentiable function and g(x) is an arbitrary
+ * non-differentiable function.  In such a situation, standard gradient descent
+ * techniques cannot work because of the non-differentiability of g(x).  To work
+ * around this, FBS takes a _forward step_ that is just a gradient descent step
+ * on f(x), and then a _backward step_ that is the _proximal operator_
+ * corresponding to g(x).  This continues until convergence.
+ *
+ * This implementation of FBS allows specification of the backward step (or
+ * proximal operator) via the `BackwardStepType` template parameter.  When using
+ * FBS, the differentiable `FunctionType` given to `Optimize()` should be f(x),
+ * *not* the combined function h(x).  g(x) should be specified by the choice of
+ * `BackwardStepType` (e.g. `L1Penalty` or `L1Maximum`).  The `Optimize()`
+ * function will then return optimized coordinates for h(x), not f(x).
+ *
+ * For more information, see the following paper:
+ *
+ * ```
+ * @article{goldstein2014field,
+ *   title={A field guide to forward-backward splitting with a FASTA
+ *       implementation},
+ *   author={Goldstein, Tom and Studer, Christoph and Baraniuk, Richard},
+ *   journal={arXiv preprint arXiv:1411.3406},
+ *   year={2014}
+ * }
+ * ```
+ */
+template<typename BackwardStepType = L1Penalty>
+class FBS
+{
+ public:
+  /**
+   * Construct the FBS optimizer with the given options, using a
+   * default-constructed BackwardStepType.
+   */
+  FBS(const double stepSize = 0.001,
+      const size_t maxIterations = 10000,
+      const double tolerance = 1e-10);
+
+  /**
+   * Construct the FBS optimizer with the given options.
+   */
+  FBS(BackwardStepType backwardStepType,
+      const double stepSize = 0.001,
+      const size_t maxIterations = 10000,
+      const double tolerance = 1e-10);
+
+  /**
+   * Optimize the given function using FBS.  The given starting
+   * point will be modified to store the finishing point of the algorithm,
+   * the final objective value is returned.
+   *
+   * FunctionType template class must provide the following functions:
+   *
+   *   double Evaluate(const arma::mat& coordinates);
+   *   void Gradient(const arma::mat& coordinates,
+   *                 arma::mat& gradient);
+   *
+   * @tparam FunctionType Type of function to be optimized.
+   * @tparam MatType Type of objective matrix.
+   * @tparam GradType Type of gradient matrix (default is MatType).
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to be optimized.
+   * @param iterate Input with starting point, and will be modified to save
+   *                the output optimial solution coordinates.
+   * @param callbacks Callback functions.
+   * @return Objective value at the final solution.
+   */
+  template<typename FunctionType, typename MatType, typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the backward step object.
+  const BackwardStepType& BackwardStep() const { return backwardStep; }
+  //! Modify the backward step object.
+  BackwardStepType& BackwardStep() { return backwardStep; }
+
+  //! Get the step size.
+  double StepSize() const { return stepSize; }
+  //! Modify the step size.
+  double& StepSize() { return stepSize; }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return maxIterations; }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return maxIterations; }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return tolerance; }
+
+ private:
+  //! The instantiated backward step object.
+  BackwardStepType backwardStep;
+
+  //! The step size for FBS steps.
+  double stepSize;
+
+  //! The maximum number of allowed iterations.
+  size_t maxIterations;
+
+  //! The tolerance for termination.
+  double tolerance;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "fbs_impl.hpp"
+
+#endif

include/ensmallen_bits/fbs/fbs_impl.hpp

@@ -0,0 +1,144 @@
+/**
+ * @file fbs_impl.hpp
+ * @author Ryan Curtin
+ *
+ * Implementation of Forward-Backward Splitting (FBS).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_FBS_IMPL_HPP
+#define ENSMALLEN_FBS_FBS_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "fbs.hpp"
+
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+//! Constructor of the FBS class.
+template<typename BackwardStepType>
+FBS<BackwardStepType>::FBS(const double stepSize,
+                           const size_t maxIterations,
+                           const double tolerance) :
+    stepSize(stepSize),
+    maxIterations(maxIterations),
+    tolerance(tolerance)
+{ /* Nothing to do. */ }
+
+template<typename BackwardStepType>
+FBS<BackwardStepType>::FBS(BackwardStepType backwardStep,
+                           const double stepSize,
+                           const size_t maxIterations,
+                           const double tolerance) :
+    backwardStep(std::move(backwardStep)),
+    stepSize(stepSize),
+    maxIterations(maxIterations),
+    tolerance(tolerance)
+{ /* Nothing to do. */ }
+
+//! Optimize the function (minimize).
+template<typename BackwardStepType>
+template<typename FunctionType, typename MatType, typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
+FBS<BackwardStepType>::Optimize(FunctionType& function,
+                                MatType& iterateIn,
+                                CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
+  FullFunctionType& f = static_cast<FullFunctionType&>(function);
+
+  // Make sure we have all necessary functions.
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // To keep track of the function value.
+  ElemType currentObjective = std::numeric_limits<ElemType>::max();
+  ElemType currentFObjective = currentObjective;
+  ElemType currentGObjective = currentObjective;
+  ElemType lastObjective = currentObjective;
+
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
+  {
+    // During this optimization, we want to optimize h(x) = f(x) + g(x).
+    // f(x) is `f`, but g(x) is specified by `BackwardStepType`.
+
+    // First compute f(x) and f'(x).
+    currentFObjective = f.EvaluateWithGradient(iterate, gradient);
+    // Now compute g(x) to get the full objective.
+    currentGObjective = backwardStep.Evaluate(iterate);
+
+    lastObjective = currentObjective;
+    currentObjective = currentFObjective + currentGObjective;
+
+    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        currentObjective, gradient, callbacks...);
+
+    // Output current objective function.
+    Info << "FBS::Optimize(): iteration " << i << ", combined objective "
+        << currentObjective << " (f(x) = " << currentFObjective << ", g(x) = "
+        << currentGObjective << ")." << std::endl;
+
+    // Check for convergence.
+    if ((i > 1) && (std::abs(currentObjective - lastObjective) < tolerance))
+    {
+      Info << "FBS::Optimize(): minimized within objective tolerance "
+          << tolerance << "; terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
+      return currentObjective;
+    }
+
+    if ((i > 1) && !std::isfinite(currentObjective))
+    {
+      Warn << "FBS::Optimize(): objective diverged to " << currentObjective
+          << "; terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
+      return currentObjective;
+    }
+
+    // Perform forward update.
+    iterate -= ElemType(stepSize) * gradient;
+    // Now perform backward step (proximal update).
+    backwardStep.ProximalStep(iterate, stepSize);
+
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
+  }
+
+  if (!terminate)
+  {
+    Info << "FBS::Optimize(): maximum iterations (" << maxIterations
+        << ") reached; terminating optimization." << std::endl;
+  }
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
+  return currentObjective;
+} // Optimize()
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/fbs/l1_constraint.hpp

@@ -0,0 +1,81 @@
+/**
+ * @file l1_constraint.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of the proximal operator for the L1 constraint.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_L1_CONSTRAINT_HPP
+#define ENSMALLEN_FBS_L1_CONSTRAINT_HPP
+
+namespace ens {
+
+/**
+ * The L1Constraint applies a specific constraint that the L1 norm of the
+ * parameters must be less than or equal to the given lambda value.
+ *
+ * Implementationally, this means that the proximal step is a projection onto
+ * the L1 ball of radius lambda.  If the constraint is satisfied, `Evaluate()`
+ * will return 0.  Otherwise, it will return infinity.
+ *
+ * This class is meant to be used with the FBS optimizer, and any other
+ * optimizer that uses a proximal operator/step.
+ */
+class L1Constraint
+{
+ public:
+  /**
+   * Construct an L1Constraint with the given maximum L1 norm for the
+   * coordinates (lambda).
+   */
+  L1Constraint(const double lambda = 0.0);
+
+  /**
+   * If the L1 norm of the coordinates is less than or equal to lambda, this
+   * returns 0.  Otherwise, it returns infinity.
+   */
+  template<typename MatType>
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const;
+
+  /**
+   * Apply a proximal step to the given `coordinates`, assuming that the forward
+   * step took a step of size `stepSize`.  This projects `coordinates` back onto
+   * the surface of the L1-ball with radius `lambda`, if the L1 norm of
+   * `coordinates` is greater than `lambda`.
+   *
+   * This may apply the proximal step multiple times to account for numerical
+   * stability issues during projection.
+   */
+  template<typename MatType>
+  void ProximalStep(MatType& coordinates, const double stepSize) const;
+
+  //! Get the L1 constraint to use when applying the proximal step.
+  double Lambda() const { return lambda; }
+  //! Modify the L1 constraint to use when applying the proximal step.
+  double& Lambda() { return lambda; }
+
+ private:
+  //! The L1 constraint value to use.
+  double lambda;
+
+  //! Helper function: extract only nonzero elements from sparse objects, or
+  //! extract the entire dense object.
+  template<typename MatType>
+  inline arma::Col<typename MatType::elem_type> ExtractNonzeros(
+      const MatType& coordinates) const;
+
+  template<typename eT>
+  inline arma::Col<eT> ExtractNonzeros(const arma::SpMat<eT>& coordinates)
+      const;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "l1_constraint_impl.hpp"
+
+#endif

include/ensmallen_bits/fbs/l1_constraint_impl.hpp

@@ -0,0 +1,201 @@
+/**
+ * @file l1_constraint_impl.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of the proximal operator for the L1 constraint.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_L1_CONSTRAINT_IMPL_HPP
+#define ENSMALLEN_FBS_L1_CONSTRAINT_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "l1_constraint.hpp"
+
+namespace ens {
+
+inline L1Constraint::L1Constraint(const double lambda) : lambda(lambda)
+{
+  // Nothing to do.
+}
+
+template<typename MatType>
+typename MatType::elem_type L1Constraint::Evaluate(const MatType& coordinates)
+    const
+{
+  typedef typename MatType::elem_type eT;
+
+  // Allow some amount of tolerance for floating-point errors.
+  const eT l1Norm = norm(vectorise(coordinates), 1);
+  if (l1Norm <= lambda)
+    return eT(0);
+  else if (std::numeric_limits<eT>::has_infinity)
+    return std::numeric_limits<eT>::infinity();
+  else
+    return std::numeric_limits<eT>::max();
+}
+
+template<typename MatType>
+void L1Constraint::ProximalStep(MatType& coordinates,
+                                const double /* stepSize */)
+    const
+{
+  // First determine whether projection is necessary.
+  if (norm(vectorise(coordinates), 1) <= lambda)
+  {
+    return;
+  }
+
+  // An empty vector can't be projected.
+  if (coordinates.n_elem == 0)
+  {
+    return;
+  }
+
+  // We use the algorithm denoted in Figure 2 of the following paper:
+  //
+  // ```
+  // @inproceedings{duchi2008efficient,
+  //   title={Efficient projections onto the L1-ball for learning in high
+  //       dimensions},
+  //   author={Duchi, John and Shalev-Shwartz, Shai and Singer, Yoram and
+  //       Chandra, Tushar},
+  //   booktitle={Proceedings of the 25th international conference on
+  //       Machine learning},
+  //   pages={272--279},
+  //   year={2008}
+  // }
+  // ```
+  //
+  // This is an iterative algorithm that has a quicksort feel, where we try to
+  // determine the "pivot" element that tells us how much we need to shrink.  In
+  // the original paper, they maintain lists indicating whether a point is above
+  // or below the pivot, but it is more expedient (and efficient) to simply copy
+  // the coordinates array and partially sort it in-place.
+
+  typedef typename MatType::elem_type eT;
+  arma::Col<eT> work = ExtractNonzeros(coordinates);
+  size_t firstUpperElement = 0;
+  size_t lastUpperElement = work.n_elem;
+  eT rho = eT(0); // This is the quantity we aim to find to perform the projection.
+  eT s = eT(0);
+
+  while (lastUpperElement > firstUpperElement)
+  {
+    const size_t k = arma::randi<size_t>(
+        arma::distr_param((int) firstUpperElement, (int) lastUpperElement - 1));
+    const eT v = work[k];
+
+    // Now perform a half-quicksort such that all elements greater than v are in
+    // the first part of the array.
+    size_t left = firstUpperElement;
+    size_t right = lastUpperElement - 1;
+    while (left <= right)
+    {
+      while ((left < lastUpperElement) && (work[left] >= v))
+        ++left;
+      while ((right > firstUpperElement) && (work[right] < v))
+        --right;
+
+      if (left >= right)
+        break;
+
+      // work[left] is less than v, and work[right] is not.  Since we want all
+      // elements greater than or equal to v on the left, swap.
+      const eT tmp = work[left];
+      work[left] = work[right];
+      work[right] = tmp;
+    }
+
+    // Now, work[0] through work[left - 1] are in the greater set G.
+    const eT sDelta = accu(work.subvec(firstUpperElement, left - 1));
+    const size_t rhoDelta = (left - firstUpperElement);
+
+    if ((s + sDelta) - ((eT) (rho + rhoDelta)) * v < eT(lambda))
+    {
+      s += sDelta;
+      rho += rhoDelta;
+      firstUpperElement = left;
+    }
+    else
+    {
+      // v was an element that was less than rho, so, shrink the array and try
+      // again with larger elements.  We actually want to shrink the array so
+      // that it does not include v, so we need to find the first element that
+      // is v (since there may be duplicates).
+      size_t firstVIndex = left - 1;
+      while ((work[firstVIndex] == v) && (firstVIndex >= firstUpperElement))
+        --firstVIndex;
+      lastUpperElement = firstVIndex + 1;
+    }
+  }
+
+  const eT theta = (s - eT(lambda)) / rho;
+  // This is a single-line implementation of the .transform() below; we use the
+  // single-line implementation so it works with Bandicoot.
+  //
+  // coordinates.transform(
+  //     [theta](eT val)
+  //     {
+  //       if (val > 0)
+  //         return std::max(val - theta, eT(0));
+  //       else
+  //         return std::min(val + theta, eT(0));
+  //     });
+  coordinates = sign(coordinates) % clamp(
+      abs(coordinates) - theta, eT(0), std::numeric_limits<eT>::max());
+
+  // Sanity check: ensure we actually ended up inside the L1 ball.  This might
+  // not happen due to floating-point inaccuracies.  If so, try again.
+  const eT newNorm = norm(coordinates, 1);
+  if (newNorm > eT(lambda) && eT(lambda) > eT(0))
+  {
+    // Shrink the L1 ball by the amount of the error.
+    eT newLambda = (eT(lambda) - 2 * (newNorm - eT(lambda)));
+    if (newLambda == eT(lambda))
+    {
+      // Make sure we at least remove a few ULPs.
+      newLambda = eT(lambda) -
+          5 * (eT(lambda) - eT(std::nexttoward(lambda, 0.0)));
+    }
+
+    L1Constraint newConstraint(newLambda);
+    newConstraint.ProximalStep(coordinates, 0.0 /* ignored */);
+  }
+}
+
+// Helper function: extract only nonzero elements from sparse objects, or
+// extract the entire dense object.
+template<typename MatType>
+inline arma::Col<typename MatType::elem_type> L1Constraint::ExtractNonzeros(
+    const MatType& coordinates) const
+{
+  typedef typename MatType::elem_type ElemType;
+  return conv_to<arma::Col<ElemType>>::from(vectorise(abs(coordinates)));
+}
+
+template<typename eT>
+inline arma::Col<eT> L1Constraint::ExtractNonzeros(
+    const arma::SpMat<eT>& coordinates) const
+{
+  arma::Col<eT> result(coordinates.n_nonzero);
+  typename arma::SpMat<eT>::const_iterator it = coordinates.begin();
+  size_t i = 0;
+  while (it != coordinates.end())
+  {
+    // Extract only nonzero values.  Note we use the absolute value because that
+    // is what the algorithm requires (not the original value).
+    result[i] = std::abs(*it);
+    ++it;
+    ++i;
+  }
+
+  return result;
+}
+
+} // namespace ens
+
+#endif