rework LLM section as per mlpack

This reverts commit db5cef9823.

.appveyor.yml

@@ -33,6 +33,7 @@ build_script:
     -DLAPACK_LIBRARY:FILEPATH=%BLAS_LIBRARY%
     -DCMAKE_PREFIX:FILEPATH="%APPVEYOR_BUILD_FOLDER%/armadillo"
     -DBUILD_SHARED_LIBS=OFF
+    -DCMAKE_POLICY_VERSION_MINIMUM=3.5
     -DCMAKE_BUILD_TYPE=Release ..
   - >
     "%MSBUILD%" "armadillo.sln"
@@ -47,6 +48,7 @@ build_script:
     -DARMADILLO_LIBRARIES=%BLAS_LIBRARY%
     -DLAPACK_LIBRARY=%BLAS_LIBRARY%
     -DBLAS_LIBRARY=%BLAS_LIBRARY%
+    -DCMAKE_POLICY_VERSION_MINIMUM=3.5
     -DCMAKE_BUILD_TYPE=Release ..
   - >
     "%MSBUILD%" "ensmallen.sln"

.github/workflows/bandicoot-test.yml

@@ -0,0 +1,54 @@
+name: Build and Test
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: ${{ matrix.config.name }}
+    runs-on: ${{ matrix.config.os }}
+    outputs:
+        tag: ${{ steps.git.outputs.tag }}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - { name: 'CUDA', os: self-hosted}
+          - { name: 'OpenCL', os: self-hosted}
+          - { name: 'CPU', os: self-hosted}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Dependencies
+      run: |
+        cd ../
+        rm -rf bandicoot-code
+        git clone --depth 1 --branch 2.1.1 https://gitlab.com/bandicoot-lib/bandicoot-code.git
+        cd bandicoot-code/
+        mkdir build/
+        cd build/
+
+        if [[ "${{ matrix.config.name }}" == "CUDA" ]]; then
+            echo "Installing Bandicoot CUDA"
+            cmake -DFIND_CUDA=ON -DFIND_OPENCL=OFF -DBUILD_TESTS=OFF ../
+            make
+        elif [[ "${{ matrix.config.name }}" == "OpenCL" ]]; then
+            echo "Installing Bandicoot OpenCL"
+            cmake -DFIND_CUDA=OFF -DFIND_OPENCL=ON -DBUILD_TESTS=OFF ../
+            make
+        fi
+
+    - name: Build ensmallen
+      run: |
+          mkdir build
+          cd build/
+
+          if [[ "${{ matrix.config.name }}" == "CPU" ]]; then
+              cmake -DUSE_BANDICOOT=OFF ..
+          else
+              cmake -DBANDICOOT_INCLUDE_DIR=../../bandicoot-code/build/tmp/include/ -DBANDICOOT_LIBRARY=../../bandicoot-code/build/libbandicoot.so ..
+          fi
+
+          make ensmallen_tests
+
+    - name: Test ensmallen
+      run: |
+          cd build/
+          ./ensmallen_tests -d yes

CMake/COOT_FindCLBLAS.cmake

@@ -0,0 +1,44 @@
+# - Find clBLAS (includes and library)
+# This module defines
+#  CLBLAS_INCLUDE_DIR
+#  CLBLAS_LIBRARIES
+#  CLBLAS_FOUND
+# also defined, but not for general use are
+#  CLBLAS_LIBRARY, where to find the library.
+
+find_path(CLBLAS_INCLUDE_DIR clBLAS.h
+/usr/include/
+/usr/local/include/
+)
+
+set(CLBLAS_NAMES ${CLBLAS_NAMES} clBLAS)
+find_library(CLBLAS_LIBRARY
+  NAMES ${CLBLAS_NAMES}
+  PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib
+  )
+
+if (CLBLAS_LIBRARY AND CLBLAS_INCLUDE_DIR)
+    set(CLBLAS_LIBRARIES ${CLBLAS_LIBRARY})
+    set(CLBLAS_FOUND "YES")
+else ()
+  set(CLBLAS_FOUND "NO")
+endif ()
+
+if (CLBLAS_FOUND)
+   if (NOT CLBLAS_FIND_QUIETLY)
+      message(STATUS "Found a clBLAS library: ${CLBLAS_LIBRARIES}")
+   endif ()
+else ()
+    if (CLBLAS_FIND_REQUIRED)
+      message(FATAL_ERROR "Could not find a clBLAS library")
+   endif ()
+endif ()
+
+# Deprecated declarations.
+set (NATIVE_CLBLAS_INCLUDE_PATH ${CLBLAS_INCLUDE_DIR} )
+get_filename_component (NATIVE_CLBLAS_LIB_PATH ${CLBLAS_LIBRARY} PATH)
+
+mark_as_advanced(
+  CLBLAS_LIBRARY
+  CLBLAS_INCLUDE_DIR
+  )

CMake/COOT_FindCLBlast.cmake

@@ -0,0 +1,44 @@
+# - Find clBlast (includes and library)
+# This module defines
+#  CLBLAST_INCLUDE_DIR
+#  CLBLAST_LIBRARIES
+#  CLBLAST_FOUND
+# also defined, but not for general use are
+#  CLBLAST_LIBRARY, where to find the library.
+
+find_path(CLBLAST_INCLUDE_DIR clblast.h
+/usr/include/
+/usr/local/include/
+)
+
+set(CLBLAST_NAMES ${CLBLAST_NAMES} clblast)
+find_library(CLBLAST_LIBRARY
+  NAMES ${CLBLAST_NAMES}
+  PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib
+  )
+
+if (CLBLAST_LIBRARY AND CLBLAST_INCLUDE_DIR)
+    set(CLBLAST_LIBRARIES ${CLBLAST_LIBRARY})
+    set(CLBLAST_FOUND "YES")
+else ()
+  set(CLBLAST_FOUND "NO")
+endif ()
+
+if (CLBLAST_FOUND)
+   if (NOT CLBLAST_FIND_QUIETLY)
+      message(STATUS "Found a clBlast library: ${CLBLAST_LIBRARIES}")
+   endif ()
+else ()
+    if (CLBLAST_FIND_REQUIRED)
+      message(FATAL_ERROR "Could not find a clBlast library")
+   endif ()
+endif ()
+
+# Deprecated declarations.
+set (NATIVE_CLBLAST_INCLUDE_PATH ${CLBLAST_INCLUDE_DIR} )
+get_filename_component (NATIVE_CLBLAST_LIB_PATH ${CLBLAST_LIBRARY} PATH)
+
+mark_as_advanced(
+  CLBLAST_LIBRARY
+  CLBLAST_INCLUDE_DIR
+  )

CMake/COOT_FindNVRTC.cmake

@@ -0,0 +1,35 @@
+# - Find clBlast (includes and library)
+# This module defines
+#  CLBLAST_INCLUDE_DIR
+#  CLBLAST_LIBRARIES
+#  CLBLAST_FOUND
+# also defined, but not for general use are
+#  CLBLAST_LIBRARY, where to find the library.
+
+set(NVRTC_NAMES ${NVRTC_NAMES} nvrtc)
+find_library(NVRTC_LIBRARY
+  NAMES ${NVRTC_NAMES}
+  PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib /usr/lib/x86_64-linux-gnu/
+  )
+
+if (NVRTC_LIBRARY)
+  set(NVRTC_LIBRARIES ${NVRTC_LIBRARY})
+  set(NVRTC_FOUND "YES")
+else ()
+  set(NVRTC_FOUND "NO")
+endif ()
+
+if (NVRTC_FOUND)
+  if (NOT NVRTC_FIND_QUIETLY)
+    message(STATUS "Found NVRTC library: ${NVRTC_LIBRARIES}")
+   endif ()
+else ()
+  if (NVRTC_FIND_REQUIRED)
+    message(FATAL_ERROR "Could not find NVRTC library")
+   endif ()
+endif ()
+
+# Deprecated declarations.
+get_filename_component (NATIVE_NVRTC_LIB_PATH ${NVRTC_LIBRARY} PATH)
+
+mark_as_advanced(NVRTC_LIBRARY)

CMake/FindBandicoot.cmake

@@ -0,0 +1,319 @@
+# - Find Bandicoot
+# Find Bandicoot: GPU accelerator add-on for the Armadillo C++ linear algebra
+# library
+#
+# Using Bandicoot:
+#  find_package(Bandicoot REQUIRED)
+#  include_directories(${BANDICOOT_INCLUDE_DIRS})
+#  add_executable(foo foo.cc)
+#  target_link_libraries(foo ${BANDICOOT_LIBRARIES})
+# This module sets the following variables:
+#  BANDICOOT_FOUND - set to true if the library is found
+#  BANDICOOT_INCLUDE_DIRS - list of required include directories
+#  BANDICOOT_LIBRARIES - list of libraries to be linked
+#  BANDICOOT_VERSION_MAJOR - major version number
+#  BANDICOOT_VERSION_MINOR - minor version number
+#  BANDICOOT_VERSION_PATCH - patch version number
+#  BANDICOOT_VERSION_STRING - version number as a string (ex: "1.0.4")
+#  BANDICOOT_VERSION_NOTE - name of the version (ex: "unstable development version")
+
+find_path(BANDICOOT_INCLUDE_DIR
+  NAMES bandicoot
+  PATHS "$ENV{ProgramFiles}/Bandicoot/include"
+  )
+
+if(BANDICOOT_INCLUDE_DIR)
+  # Extract version information.
+  file(READ "${BANDICOOT_INCLUDE_DIR}/bandicoot_bits/coot_version.hpp" _bandicoot_HEADER_CONTENTS)
+  string(REGEX REPLACE ".*#define COOT_VERSION_MAJOR ([0-9]+).*" "\\1" BANDICOOT_VERSION_MAJOR "${_bandicoot_HEADER_CONTENTS}")
+  string(REGEX REPLACE ".*#define COOT_VERSION_MINOR ([0-9]+).*" "\\1" BANDICOOT_VERSION_MINOR "${_bandicoot_HEADER_CONTENTS}")
+  string(REGEX REPLACE ".*#define COOT_VERSION_PATCH ([0-9]+).*" "\\1" BANDICOOT_VERSION_PATCH "${_bandicoot_HEADER_CONTENTS}")
+  string(REGEX REPLACE ".*#define COOT_VERSION_NOTE\ +\"([0-9a-zA-Z\ _-]+)\".*" "\\1" BANDICOOT_VERSION_NOTE "${_bandicoot_HEADER_CONTENTS}")
+
+  set(BANDICOOT_VERSION_STRING "${BANDICOOT_VERSION_MAJOR}.${BANDICOOT_VERSION_MINOR}.${BANDICOOT_VERSION_PATCH}")
+endif ()
+
+# Determine what support libraries are being used, and whether or not we need to
+# link against them.  We need to look in config.hpp.
+set(SUPPORT_INCLUDE_DIRS "")
+set(SUPPORT_LIBRARIES "")
+set(COOT_NEED_LIBRARY true) # Assume true.
+if(EXISTS "${BANDICOOT_INCLUDE_DIR}/bandicoot_bits/config.hpp")
+  file(READ "${BANDICOOT_INCLUDE_DIR}/bandicoot_bits/config.hpp" _bandicoot_CONFIG_CONTENTS)
+  # COOT_USE_WRAPPER
+  string(REGEX MATCH "\r?\n[\t ]*#define[ \t]+COOT_USE_WRAPPER[ \t]*\r?\n" COOT_USE_WRAPPER "${_bandicoot_CONFIG_CONTENTS}")
+
+  # COOT_USE_OPENCL
+  string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]COOT_USE_OPENCL[)][\t ]*\r?\n[\t
+  ]*#define[ \t]+COOT_USE_OPENCL[ \t]*\r?\n" COOT_USE_OPENCL "${_bandicoot_CONFIG_CONTENTS}")
+
+  # COOT_USE_CUDA
+  string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]COOT_USE_CUDA[)][\t ]*\r?\n[\t
+  ]*#define[ \t]+COOT_USE_CUDA[ \t]*\r?\n" COOT_USE_CUDA "${_bandicoot_CONFIG_CONTENTS}")
+
+  # COOT_USE_LAPACK
+  string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]COOT_USE_LAPACK[)][\t ]*\r?\n[\t ]*#define[ \t]+COOT_USE_LAPACK[ \t]*\r?\n" COOT_USE_LAPACK "${_bandicoot_CONFIG_CONTENTS}")
+
+  # COOT_USE_BLAS
+  string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]COOT_USE_BLAS[)][\t ]*\r?\n[\t ]*#define[ \t]+COOT_USE_BLAS[ \t]*\r?\n" COOT_USE_BLAS "${_bandicoot_CONFIG_CONTENTS}")
+
+  # If we aren't wrapping, things get a little more complex.
+  if(NOT COOT_USE_WRAPPER)
+    set(COOT_NEED_LIBRARY false)
+    message(STATUS "COOT_USE_WRAPPER is not defined, so all dependencies of "
+                   "Bandicoot must be manually linked.")
+
+    set(HAVE_OPENCL false)
+    set(HAVE_CUDA   false)
+    set(HAVE_LAPACK false)
+    set(HAVE_BLAS   false)
+
+    # Search for OpenCL.
+    if (NOT "${COOT_USE_OPENCL}" STREQUAL "" AND NOT HAVE_OPENCL)
+      set(OpenCL_FIND_QUIETLY true)
+      include(FindOpenCL)
+
+      if (OpenCL_FOUND)
+        message(STATUS "OpenCL includes: ${OpenCL_INCLUDE_DIRS}")
+        message(STATUS "OpenCL libraries: ${OpenCL_LIBRARIES}")
+
+        set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+            "${OpenCL_INCLUDE_DIRS}")
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${OpenCL_LIBRARIES}")
+        set(HAVE_OPENCL true)
+      endif ()
+
+      # Search for clBLAS.
+      set(CLBLAS_FIND_QUIETLY true)
+      include(COOT_FindCLBLAS)
+
+      if (CLBLAS_FOUND)
+        message(STATUS "clBLAS includes: ${CLBLAS_INCLUDE_DIR}")
+        message(STATUS "clBLAS libraries: ${CLBLAS_LIBRARIES}")
+
+        set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+            "${CLBLAS_INCLUDE_DIR}")
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${CLBLAS_LIBRARIES}")
+        set(HAVE_CLBLAS true)
+      endif ()
+    endif ()
+
+    # Search for CUDA.
+    if (NOT COOT_USE_CUDA AND NOT HAVE_CUDA)
+      # FindCUDA is deprecated since version 3.10 and replaced with
+      # FindCUDAToolkit wich was added in CMake 3.17.
+      message(STATUS "${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}")
+      if ("${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" LESS "3.67")
+        set(CUDA_FIND_QUIETLY true)
+        find_package(CUDA)
+
+        if (CUDA_FOUND)
+	  message(STATUS "CUDA includes: ${CUDA_INCLUDE_DIRS}")
+	  message(STATUS "CUDA libraries: ${CUDA_LIBRARIES}")
+
+          # We also need NVRTC and also libcuda itself, which the old FindCUDA package do not find.
+          find_library(CUDA_cuda_LIBRARY cuda
+              HINTS ${CUDA_TOOLKIT_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64)
+          find_library(CUDA_nvrtc_LIBRARY nvrtc
+              HINTS ${CUDA_TOOLKIT_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64)
+
+          include(COOT_FindNVRTC)
+
+          if (NVRTC_FOUND)
+            message(STATUS "NVRTC libraries: ${NVRTC_LIBRARIES}")
+            set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${NVRTC_LIBRARIES}")
+          endif ()
+
+          set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+              "${CUDA_INCLUDE_DIRS}")
+          set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}"
+              "${CUDA_LIBRARIES}"
+              "${CUDA_nvrtc_LIBRARY}"
+              "${CUDA_CUDA_LIBRARY}"
+              "${CUDA_CUBLAS_LIBRARIES}"
+              "${CUDA_curand_LIBRARY}"
+              "${CUDA_cusolver_LIBRARY}")
+          set(CUDA_INCLUDE_DIRS "")
+          set(HAVE_CUDA true)
+
+        endif ()
+      else ()
+        set(CUDA_TOOLKIT_FIND_QUIETLY true)
+        find_package(CUDAToolkit REQUIRED)
+
+        if (CUDAToolkit_FOUND)
+	  message(STATUS "CUDA includes: ${CUDAToolkit_INCLUDE_DIRS}")
+	  message(STATUS "CUDA libraries: ${CUDAToolkit_LIBRARY_DIR}")
+
+	  set(CUDA_LIBRARIES CUDA::cudart CUDA::cuda_driver)
+          set(CUDA_CUBLAS_LIBRARIES CUDA::cublas)
+          set(CUDA_curand_LIBRARY CUDA::curand)
+          set(CUDA_cusolver_LIBRARY CUDA::cusolver)
+          set(CUDA_nvrtc_LIBRARY CUDA::nvrtc)
+
+          set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+              "${CUDAToolkit_INCLUDE_DIRS}")
+          set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}"
+              CUDA_LIBRARIES
+	      CUDA_CUBLAS_LIBRARIES
+	      CUDA_curand_LIBRARY
+	      CUDA_cusolver_LIBRARY
+	      CUDA_nvrtc_LIBRARY)
+          set(HAVE_CUDA true)
+        endif()
+      endif ()
+    endif ()
+
+    # Search for LAPACK/BLAS (or replacement).
+    if ((NOT "${COOT_USE_LAPACK}" STREQUAL "") AND
+      (NOT "${COOT_USE_BLAS}" STREQUAL ""))
+      # In order of preference: MKL, ACML, OpenBLAS, ATLAS
+      set(MKL_FIND_QUIETLY true)
+      include(ARMA_FindMKL)
+      set(ACMLMP_FIND_QUIETLY true)
+      include(ARMA_FindACMLMP)
+      set(ACML_FIND_QUIETLY true)
+      include(ARMA_FindACML)
+
+      if (MKL_FOUND)
+        message(STATUS "Using MKL for LAPACK/BLAS: ${MKL_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${MKL_LIBRARIES}")
+        set(HAVE_LAPACK true)
+        set(HAVE_BLAS   true)
+      elseif (ACMLMP_FOUND)
+        message(STATUS "Using multi-core ACML libraries for LAPACK/BLAS:
+            ${ACMLMP_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${ACMLMP_LIBRARIES}")
+        set(HAVE_LAPACK true)
+        set(HAVE_BLAS   true)
+      elseif (ACML_FOUND)
+        message(STATUS "Using ACML for LAPACK/BLAS: ${ACML_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${ACML_LIBRARIES}")
+        set(HAVE_LAPACK true)
+        set(HAVE_BLAS   true)
+      endif ()
+    endif ()
+
+    # If we haven't found BLAS, try.
+    if (NOT "${COOT_USE_BLAS}" STREQUAL "" AND NOT HAVE_BLAS)
+      # Search for BLAS.
+      set(OpenBLAS_FIND_QUIETLY false)
+      include(ARMA_FindOpenBLAS)
+      set(CBLAS_FIND_QUIETLY true)
+      include(ARMA_FindCBLAS)
+      set(BLAS_FIND_QUIETLY true)
+      include(ARMA_FindBLAS)
+
+      if (OpenBLAS_FOUND)
+        # Warn if ATLAS is found also.
+        if (CBLAS_FOUND)
+          message(STATUS "Warning: both OpenBLAS and ATLAS have been found; "
+              "ATLAS will not be used.")
+        endif ()
+        message(STATUS "Using OpenBLAS for BLAS: ${OpenBLAS_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${OpenBLAS_LIBRARIES}")
+        set(HAVE_BLAS true)
+      elseif (CBLAS_FOUND)
+        message(STATUS "Using ATLAS for BLAS: ${CBLAS_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${CBLAS_LIBRARIES}")
+        set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+            "${CBLAS_INCLUDE_DIR}")
+        set(HAVE_BLAS true)
+      elseif (BLAS_FOUND)
+        message(STATUS "Using standard BLAS: ${BLAS_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${BLAS_LIBRARIES}")
+        set(HAVE_BLAS true)
+      endif ()
+    endif ()
+
+    # If we haven't found LAPACK, try.
+    if (NOT "${COOT_USE_LAPACK}" STREQUAL "" AND NOT HAVE_LAPACK)
+      # Search for LAPACK.
+      set(CLAPACK_FIND_QUIETLY true)
+      include(ARMA_FindCLAPACK)
+      set(LAPACK_FIND_QUIETLY true)
+      include(ARMA_FindLAPACK)
+
+      # Only use ATLAS if OpenBLAS isn't being used.
+      if (CLAPACK_FOUND AND NOT OpenBLAS_FOUND)
+        message(STATUS "Using ATLAS for LAPACK: ${CLAPACK_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${CLAPACK_LIBRARIES}")
+        set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}"
+            "${CLAPACK_INCLUDE_DIR}")
+        set(HAVE_LAPACK true)
+      elseif (LAPACK_FOUND)
+        message(STATUS "Using standard LAPACK: ${LAPACK_LIBRARIES}")
+
+        set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${LAPACK_LIBRARIES}")
+        set(HAVE_LAPACK true)
+      endif ()
+    endif ()
+
+    if (NOT "${COOT_USE_LAPACK}" STREQUAL "" AND NOT HAVE_LAPACK)
+      message(FATAL_ERROR "Cannot find LAPACK library, but COOT_USE_LAPACK is "
+          "set. Try specifying LAPACK libraries manually by setting the "
+          "LAPACK_LIBRARY variable.")
+    endif ()
+
+    if (NOT "${COOT_USE_BLAS}" STREQUAL "" AND NOT HAVE_BLAS)
+      message(FATAL_ERROR "Cannot find BLAS library, but COOT_USE_BLAS is set. "
+          "Try specifying BLAS libraries manually by setting the BLAS_LIBRARY "
+          "variable.")
+    endif ()
+
+  endif()
+else()
+  message(STATUS "${BANDICOOT_INCLUDE_DIR}/bandicoot_bits/config.hpp not "
+      "found!  Cannot determine what to link against.")
+endif()
+
+if (COOT_NEED_LIBRARY)
+  # UNIX paths are standard, no need to write.
+  find_library(BANDICOOT_LIBRARY
+    NAMES bandicoot
+    PATHS "$ENV{ProgramFiles}/Bandicoot/lib"  "$ENV{ProgramFiles}/Bandicoot/lib64" "$ENV{ProgramFiles}/Bandicoot"
+    )
+
+  # Checks 'REQUIRED', 'QUIET' and versions.
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(Bandicoot
+    REQUIRED_VARS BANDICOOT_LIBRARY BANDICOOT_INCLUDE_DIR
+    VERSION_VAR BANDICOOT_VERSION_STRING)
+else ()
+  # Checks 'REQUIRED', 'QUIET' and versions.
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(Bandicoot
+    REQUIRED_VARS BANDICOOT_INCLUDE_DIR
+    VERSION_VAR BANDICOOT_VERSION_STRING)
+endif ()
+
+if (BANDICOOT_FOUND)
+  # Also include support include directories.
+  set(BANDICOOT_INCLUDE_DIRS ${BANDICOOT_INCLUDE_DIR} ${SUPPORT_INCLUDE_DIRS})
+  # Also include support libraries to link against.
+  if (COOT_NEED_LIBRARY)
+    set(BANDICOOT_LIBRARIES ${BANDICOOT_LIBRARY} ${SUPPORT_LIBRARIES})
+  else ()
+    set(BANDICOOT_LIBRARIES ${SUPPORT_LIBRARIES})
+  endif ()
+  message(STATUS "Bandicoot libraries: ${BANDICOOT_LIBRARIES}")
+  message(STATUS "Bandicoot includes: ${BANDICOOT_INCLUDE_DIR}")
+endif ()
+
+# Hide internal variables
+mark_as_advanced(
+  BANDICOOT_INCLUDE_DIR
+  BANDICOOT_LIBRARIES)
+
+if (BANDICOOT_FOUND AND NOT TARGET Bandicoot::Bandicoot)
+  add_library(Bandicoot::Bandicoot INTERFACE IMPORTED)
+  set_target_properties(Bandicoot::Bandicoot PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${BANDICOOT_INCLUDE_DIR}"
+      INTERFACE_LINK_LIBRARIES "${BANDICOOT_LIBRARIES}")
+endif()

CMake/FindCBLAS.cmake

@@ -0,0 +1,75 @@
+
+# ==================================================================================================
+# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
+# project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
+# width of 100 characters per line.
+#
+# Author(s):
+#   Cedric Nugteren <www.cedricnugteren.nl>
+#
+# ==================================================================================================
+#
+# Defines the following variables:
+#   CBLAS_FOUND          Boolean holding whether or not the Netlib BLAS library was found
+#   CBLAS_INCLUDE_DIRS   The Netlib BLAS include directory
+#   CBLAS_LIBRARIES      The Netlib BLAS library
+#
+# In case BLAS is not installed in the default directory, set the CBLAS_ROOT variable to point to
+# the root of BLAS, such that 'cblas.h' can be found in $CBLAS_ROOT/include. This can either be
+# done using an environmental variable (e.g. export CBLAS_ROOT=/path/to/BLAS) or using a CMake
+# variable (e.g. cmake -DCBLAS_ROOT=/path/to/BLAS ..).
+#
+# ==================================================================================================
+
+# Sets the possible install locations
+set(CBLAS_HINTS
+  ${CBLAS_ROOT}
+  $ENV{CBLAS_ROOT}
+)
+set(CBLAS_PATHS
+  /usr
+  /usr/local
+  /usr/local/opt
+  /System/Library/Frameworks
+)
+
+# Finds the include directories
+find_path(CBLAS_INCLUDE_DIRS
+  NAMES cblas.h
+  HINTS ${CBLAS_HINTS}
+  PATH_SUFFIXES
+    include inc include/x86_64 include/x64
+    openblas/include include/blis blis/include blis/include/blis
+    Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current/Headers
+  PATHS ${CBLAS_PATHS}
+  DOC "Netlib BLAS include header cblas.h"
+)
+mark_as_advanced(CBLAS_INCLUDE_DIRS)
+
+# Finds the library
+find_library(CBLAS_LIBRARIES
+  NAMES cblas blas blis openblas accelerate
+  HINTS ${CBLAS_HINTS}
+  PATH_SUFFIXES
+    lib lib64 lib/x86_64 lib/x64 lib/x86 lib/Win32 lib/import lib64/import
+    openblas/lib blis/lib lib/atlas-base
+  PATHS ${CBLAS_PATHS}
+  DOC "Netlib BLAS library"
+)
+mark_as_advanced(CBLAS_LIBRARIES)
+
+# ==================================================================================================
+
+# Notification messages
+if(NOT CBLAS_INCLUDE_DIRS)
+    message(STATUS "Could NOT find 'cblas.h', install a CPU Netlib BLAS or set CBLAS_ROOT")
+endif()
+if(NOT CBLAS_LIBRARIES)
+    message(STATUS "Could NOT find a CPU Netlib BLAS library, install it or set CBLAS_ROOT")
+endif()
+
+# Determines whether or not BLAS was found
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(CBLAS DEFAULT_MSG CBLAS_INCLUDE_DIRS CBLAS_LIBRARIES)
+
+# ==================================================================================================

CMakeLists.txt

@@ -10,12 +10,17 @@ project(ensmallen
 
 # Configurable options for CMake.
 option(USE_OPENMP "If available, use OpenMP for parallelization." ON)
+option(USE_BANDICOOT "If available, build against Bandicoot for GPU support." ON)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake")
 
-# Set required C++ standard to C++14.
-set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
+# Set minimum required C++ standard to C++14.
+if (NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 14)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+elseif (${CMAKE_CXX_STANDARD} LESS 14)
+  message(FATAL_ERROR "ensmallen requires C++14 or newer!")
+endif ()
 
 # Extract version from sources.
 set(ENSMALLEN_VERSION_FILE_NAME "${PROJECT_SOURCE_DIR}/include/ensmallen_bits/ens_version.hpp")
@@ -59,6 +64,17 @@ if(USE_OPENMP)
   target_link_libraries(ensmallen INTERFACE OpenMP::OpenMP_CXX)
 endif()
 
+if(USE_BANDICOOT)
+  # Find Bandicoot and link it.
+  find_package(Bandicoot 2.1.0)
+  if(BANDICOOT_FOUND)
+    target_link_libraries(ensmallen INTERFACE Bandicoot::Bandicoot)
+    target_include_directories(ensmallen INTERFACE ${BANDICOOT_INCLUDE_DIR})
+
+    add_definitions(-DENS_USE_COOT)
+  endif()
+endif()
+
 # Find Armadillo and link it.
 find_package(Armadillo 10.8.2 REQUIRED)
 target_link_libraries(ensmallen INTERFACE Armadillo::Armadillo)

CONTRIBUTING.md

@@ -8,9 +8,27 @@ you have an improvement you would like to see, we would love to include it!
 
 The ensmallen maintainer community overlaps heavily with the
 [mlpack](https://github.com/mlpack/mlpack) community, so development discussions
-can happen either here on Github, on the [mlpack mailing
-list](http://lists.mlpack.org/mailman/listinfo/mlpack), or in the #mlpack
-IRC channel on irc.freenode.net.
+can happen either here on Github or in the `#mlpack:matrix.org` channel on
+[Matrix](https://www.matrix.org/).  See
+[here](https://www.mlpack.org/doc/developer/community.html) for more
+information.
+
+## Usage of LLMs / Coding Assistants
+
+Each contributor needs to understand the content they are proposing for ensmallen,
+and how the proposed content changes and/or expands functionality in ensmallen.
+Content includes source code, documentation, and other material such as images
+and datasets.  The pull request must justify the proposed content in the
+description.  Using coding assistants or tools like Large Language Models (LLMs) 
+does not grant additional privileges or reduce our expectations.
+
+Each contributor is responsible for the proposed content, regardless of where
+the content came from.  The responsibility includes ensuring that the content
+can be submitted to ensmallen and does not violate intellectual property rights
+such as copyright(s).  Source code in ensmallen is licensed under the BSD license;
+see `LICENSE.txt` for details.
+
+## Pull request process
 
 Once a pull request is submitted, it must be reviewed and approved before a
 merge, to ensure that:
@@ -108,8 +126,8 @@ $ cd ensmallen
 
 # - or -
 
-$ wget http://ensmallen.org/files/ensmallen-2.22.2.tar.gz
-$ tar -xvzpf ensmallen-2.22.2.tar.gz
+$ wget http://ensmallen.org/files/ensmallen-3.11.0.tar.gz
+$ tar -xvzpf ensmallen-3.11.0.tar.gz
 $ cd ensmallen-latest
 ```
 

HISTORY.md

@@ -1,3 +1,79 @@
+### ensmallen ?.??.?: "???"
+###### ????-??-??
+
+### ensmallen 3.11.0: "Sunny Day"
+###### 2025-12-15
+ * Refactor `GradientDescent` into
+   `GradientDescentType<UpdatePolicyType, DecayPolicyType>` and
+   add the `DeltaBarDelta` and `MomentumDeltaBarDelta` optimizers
+   ([#440](https://github.com/mlpack/ensmallen/pull/440)).
+
+ * Fix an off-by-one bug where the actual number of executed iterations was one
+   fewer than the specified `maxIterations`
+   ([#443](https://github.com/mlpack/ensmallen/pull/443)).
+
+### ensmallen 3.10.0: "Unexpected Rain"
+###### 2025-09-25
+ * SGD-like optimizers now all divide the step size by the batch size so that
+   step sizes don't need to be tuned in addition to batch sizes.  If you require
+   behavior from ensmallen 2, define the `ENS_OLD_SEPARABLE_STEP_BEHAVIOR` macro
+   before including `ensmallen.hpp`
+   ([#431](https://github.com/mlpack/ensmallen/pull/431)).
+
+ * Remove deprecated `ParetoFront()` and `ParetoSet()` from multi-objective
+   optimizers ([#435](https://github.com/mlpack/ensmallen/pull/435)).  Instead,
+   pass objects to the `Optimize()` function; see the documentation for each
+   multi-objective optimizer for more details.  A typical transition will change
+   code like:
+
+    ```c++
+    optimizer.Optimize(objectives, coordinates);
+    arma::cube paretoFront = optimizer.ParetoFront();
+    arma::cube paretoSet = optimizer.ParetoSet();
+    ```
+
+   to instead gather the Pareto front and set in the call:
+
+    ```c++
+    arma::cube paretoFront, paretoSet;
+    optimizer.Optimize(objectives, coordinates, paretoFront, paretoSet);
+    ```
+
+ * Remove deprecated constructor for Active CMA-ES that takes `lowerBound` and
+   `upperBound` ([#435](https://github.com/mlpack/ensmallen/pull/435)).
+   Instead, pass an instantiated `BoundaryBoxConstraint` to the constructor.  A
+   typical transition will change code like:
+
+    ```c++
+    ActiveCMAES<FullSelection, BoundaryBoxConstraint> opt(lambda,
+        lowerBound, upperBound, ...);
+    ```
+
+   into
+
+    ```c++
+    ActiveCMAES<FullSelection, BoundaryBoxConstraint> opt(lambda,
+        BoundaryBoxConstraint(lowerBound, upperBound), ...);
+    ```
+
+ * Add proximal gradient optimizers for L1-constrained and other related
+   problems: `FBS`, `FISTA`, and `FASTA`
+   ([#427](https://github.com/mlpack/ensmallen/pull/427)).  See the
+   documentation for more details.
+
+ * The `Lambda()` and `Sigma()` functions of the `AugLagrangian` optimizer,
+   which could be used to retrieve the Lagrange multipliers and penalty
+   parameter after optimization, are now deprecated
+   ([#439](https://github.com/mlpack/ensmallen/pull/439)).  Instead, pass a
+   vector and a double to the `Optimize()` function directly:
+
+    ```c++
+    augLag.Optimize(function, coordinates, lambda, sigma)
+    ```
+
+   and these will be filled with the final Lagrange multiplier estimates and
+   penalty parameters.
+
 ### ensmallen 2.22.2: "E-Bike Excitement"
 ###### 2025-04-30
  * Fix include statement in `tests/de_test.cpp`

README.md

@@ -22,12 +22,12 @@ Documentation and downloads: https://ensmallen.org
 
 ### Installation
 
-ensmallen can be installed in several ways: either manually or via cmake, 
+ensmallen can be installed in several ways: either manually or via cmake,
 with or without root access.
 
-The cmake based installation will check the requirements 
-and optionally build the tests. If cmake 3.3 (or a later version) 
-is not already available on your system, it can be obtained 
+The cmake based installation will check the requirements
+and optionally build the tests. If cmake 3.3 (or a later version)
+is not already available on your system, it can be obtained
 from [cmake.org](https://cmake.org).
 
 Example cmake based installation with root access:
@@ -39,7 +39,7 @@ cmake ..
 sudo make install
 ```
 
-Example cmake based installation without root access, 
+Example cmake based installation without root access,
 installing into `/home/blah/` (adapt as required):
 
 ```
@@ -49,7 +49,7 @@ cmake .. -DCMAKE_INSTALL_PREFIX:PATH=/home/blah/
 make install
 ```
 
-The above will create a directory named `/home/blah/include/` 
+The above will create a directory named `/home/blah/include/`
 and place all ensmallen headers there.
 
 To optionally build and run the tests
@@ -61,10 +61,10 @@ make ensmallen_tests
 ./ensmallen_tests --durations yes
 ```
 
-Manual installation involves simply copying the `include/ensmallen.hpp` header 
-***and*** the associated `include/ensmallen_bits` directory to a location 
+Manual installation involves simply copying the `include/ensmallen.hpp` header
+***and*** the associated `include/ensmallen_bits` directory to a location
 such as `/usr/include/` which is searched by your C++ compiler.
-If you can't use `sudo` or don't have write access to `/usr/include/`, 
+If you can't use `sudo` or don't have write access to `/usr/include/`,
 use a directory within your own home directory (eg. `/home/blah/include/`).
 
 
@@ -73,11 +73,11 @@ use a directory within your own home directory (eg. `/home/blah/include/`).
 If you have installed ensmallen in a standard location such as `/usr/include/`:
 
     g++ prog.cpp -o prog -O2 -larmadillo
-    
-If you have installed ensmallen in a non-standard location, 
-such as `/home/blah/include/`, you will need to make sure 
-that your C++ compiler searches `/home/blah/include/` 
-by explicitly specifying the directory as an argument/option. 
+
+If you have installed ensmallen in a non-standard location,
+such as `/home/blah/include/`, you will need to make sure
+that your C++ compiler searches `/home/blah/include/`
+by explicitly specifying the directory as an argument/option.
 For example, using the `-I` switch in gcc and clang:
 
     g++ prog.cpp -o prog -O2 -I /home/blah/include/ -larmadillo
@@ -85,7 +85,7 @@ For example, using the `-I` switch in gcc and clang:
 
 ### Example Optimization
 
-See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer 
+See [`example.cpp`](example.cpp) for example usage of the L-BFGS optimizer
 in a linear regression setting.
 
 
@@ -103,8 +103,8 @@ Please cite the following paper if you use ensmallen in your research and/or
 software. Citations are useful for the continued development and maintenance of
 the library.
 
-* Ryan R. Curtin, Marcus Edel, Rahul Ganesh Prabhu, Suryoday Basak, Zhihao Lou, Conrad Sanderson.  
-  [The ensmallen library for flexible numerical optimization](https://jmlr.org/papers/volume22/20-416/20-416.pdf).  
+* Ryan R. Curtin, Marcus Edel, Rahul Ganesh Prabhu, Suryoday Basak, Zhihao Lou, Conrad Sanderson.
+  [The ensmallen library for flexible numerical optimization](https://jmlr.org/papers/volume22/20-416/20-416.pdf).
   Journal of Machine Learning Research, Vol. 22, No. 166, 2021.
 
 ```

doc/callbacks.md

@@ -1,10 +1,10 @@
-Callbacks in ensmallen are methods that are called at various states during the
-optimization process, which can be used to implement and control behaviors such
-as:
+Callbacks in ensmallen are methods that are called at various stages of the
+optimization process.  These can be used to print information about the optimization, modify behavior of the optimization, or a wide range of other possibilities.  Some examples of what callbacks can be used for include:
 
 * Changing the learning rate.
-* Printing of the current objective.
-* Sending a message when the optimization hits a specific state such us a minimal objective.
+* Printing the current objective.
+* Sending a message when the optimization hits a specific state such us a
+  minimal objective.
 
 Callbacks can be passed as an argument to the `Optimize()` function:
 
@@ -73,6 +73,27 @@ std::cout << callback.BestObjective() << std::endl;
 
 </details>
 
+Numerous implemented and ready-to-use callbacks are included with ensmallen, and
+it is also easy to write a custom callback.
+
+ * [`EarlyStopAtMinLoss`](#earlystopatminloss): stop the optimization if no
+   improvement has been made
+ * [`GradClipByNorm`](#gradclipbynorm): reduce the norm of the gradient to
+   prevent the exploding gradient problem
+ * [`GradClipByValue`](#gradclipbyvalue): clip the gradient to specified minimum
+   and maximum values
+ * [`PrintLoss`](#printloss): print the objective at each iteration to a
+   specified stream
+ * [`ProgressBar`](#progressbar): print a progress bar to the screen at each
+   iteration
+ * [`Report`](#report): print a report at the end of optimization
+ * [`StoreBestCoordinates`](#storebestcoordinates): store the coordinates that
+   give the best objective value at the end of an epoch
+ * [`TimerStop`](#timerstop): stop the optimization after a given amount of time
+
+A [guide for implementing custom callbacks](#custom-callbacks) is below, and a
+few [example custom callbacks](#custom-callback-examples) are given too.
+
 ## Built-in Callbacks
 
 ### EarlyStopAtMinLoss
@@ -154,7 +175,7 @@ instability. This can happen due to:
 * A high learning rate, leading to large gradient updates.
 * Poorly scaled datasets, resulting in significant variance between data points.
 * A loss function that generates disproportionately large error values.
-    
+
 Common solutions for this problem are:
 
 #### GradClipByNorm
@@ -412,174 +433,304 @@ std::cout << "The optimized model found by AdaDelta has the "
 
 </details>
 
-## Callback States
+### TimerStop
 
-Callbacks are called at several states during the optimization process:
+Callback that stops optimization after a certain amount of time has elapsed.
 
-* At the beginning and end of the optimization process.
-* After any call to `Evaluate()` and `EvaluateConstraint()`.
-* After any call to `Gradient()` and `GradientConstraint()`.
-* At the start and end of an epoch.
+#### Constructors
 
-Each callback provides optimization-relevant information that can be accessed or
-modified.
+ * `TimerStop(`_`seconds`_`)`
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+// Limit optimization to 15 seconds.
+optimizer.Optimize(f, coordinates, TimerStop(15));
+```
+
+</details>
+
+## Custom Callbacks
+
+Custom callbacks can be easily implemented by creating a class and simply
+implementing functions for each individual callback that you are interested in
+handling, plus any other functionality you might need (e.g. constructors,
+accessors).
+
+Thus, when writing a custom callback, start with an empty class like this:
+
+```c++
+class CustomCallback
+{
+ public:
+  // Add individual callback handlers that you are interested in handling!
+};
+```
+
+and add any of the individual callback handler functions described below.
 
 ### BeginOptimization
 
-Called at the beginning of the optimization process.
+Called at the beginning of the optimization process.  Add this function to your
+callback class with your desired implementation:
 
- * `void BeginOptimization(`_`optimizer, function, coordinates`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  void BeginOptimization(OptimizerType& optimizer,
+                         FunctionType& function,
+                         MatType& coordinates);
+```
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function to be optimized (e.g. the first argument given to
+   `optimizer.Optimize()`.
+ * `coordinates`: the current coordinates for optimization; since optimization
+   is just beginning, this is the exact same matrix given to `Optimize()`.
 
 ### EndOptimization
 
-Called at the end of the optimization process.
+Called at the end of the optimization process.  Add this function to your
+callback class with your desired implementation:
 
- * `void EndOptimization(`_`optimizer, function, coordinates`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  void EndOptimization(OptimizerType& optimizer,
+                       FunctionType& function,
+                       MatType& coordinates);
+```
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that has been optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the final coordinates for optimization; since optimization is
+   ending, these are the same values that will be in the resulting matrix after
+   `Optimize()` finishes.
 
 ### Evaluate
 
-Called after any call to `Evaluate()`.
+Called after any call to `Evaluate()` or `EvaluateWithGradient()`.  Add this
+function to your callback class with your desired implementation:
 
- * `bool Evaluate(`_`optimizer, function, coordinates, objective`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool Evaluate(OptimizerType& optimizer,
+                FunctionType& function,
+                const MatType& coordinates,
+                const double objective);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates with which `function.Evaluate()` was called.
+ * `objective`: the result of `function.Evaluate(coordinates)`.
 
 If the callback returns `true`, the optimization will be terminated.
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `double` | **`objective`** | Objective value of the current point. |
-
 ### EvaluateConstraint
 
-Called after any call to `EvaluateConstraint()`.
+Called after any call to `EvaluateConstraint()`, for
+[constrained functions](#constrained-functions).  Add this function to your
+callback class with your desired implementation:
 
- * `bool EvaluateConstraint(`_`optimizer, function, coordinates, constraint, constraintValue`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool EvaluateConstraint(OptimizerType& optimizer,
+                          FunctionType& function,
+                          const MatType& coordinates,
+                          const size_t constraintIndex,
+                          const double constraintValue);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates with which `function.EvaluateConstraint()` was
+   called.
+ * `constraintIndex`: the index of the constraint that was evaluated
+ * `constraintValue`: the result of
+   `function.EvaluateConstraint(coordinates, constraintIndex)`.
 
 If the callback returns `true`, the optimization will be terminated.
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `size_t` | **`constraint`** | The index of the constraint. |
-| `double` | **`constraintValue`** | Constraint value of the current point. |
-
 ### Gradient
 
-Called after any call to `Gradient()`.
+Called after any call to `Gradient()` or `EvaluateWithGradient()`.  Add this
+function to your callback class with your desired implementation:
 
- * `bool Gradient(`_`optimizer, function, coordinates, gradient`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  bool Gradient(OptimizerType& optimizer,
+                FunctionType& function,
+                const MatType& coordinates,
+                GradType& gradient);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates with which `function.Gradient()` was called.
+ * `gradient`: the computed gradient (can be modified!).
 
 If the callback returns `true`, the optimization will be terminated.
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `GradType` | **`gradient`** | Matrix that holds the gradient. |
-
 ### GradientConstraint
 
-Called after any call to `GradientConstraint()`.
+Called after any call to `GradientConstraint()` for
+[constrained functions](#constrained-functions).  Add this function to your
+callback class with your desired implementation:
 
- * `bool GradientConstraint(`_`optimizer, function, coordinates, constraint, gradient`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  bool GradientConstraint(OptimizerType& optimizer,
+                          FunctionType& function,
+                          const MatType& coordinates,
+                          const size_t constraintIndex,
+                          GradType& constraintGradient);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates with which `function.GradientConstraint()` was
+   called.
+ * `constraintIndex`: the index of the constraint whose gradient was computed.
+ * `constraintGradient`: the computed result of
+   `function.GradientConstraint()`.
 
 If the callback returns `true`, the optimization will be terminated.
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `size_t` | **`constraint`** | The index of the constraint. |
-| `GradType` | **`gradient`** | Matrix that holds the gradient. |
-
 ### BeginEpoch
 
-Called at the beginning of a pass over the data. The objective may be exact or
-an estimate depending on `exactObjective` value.
+Called at the beginning of a pass over the data, for
+[separable functions](#separable-functions). The objective may be exact or
+an estimate depending on the optimizer's `ExactObjective()` value.  Add this
+function to your callback class with your desired implementation:
 
- * `bool BeginEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool BeginEpoch(OptimizerType& optimizer,
+                  FunctionType& function,
+                  const MatType& coordinates,
+                  const size_t epoch,
+                  const double objective);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates at the start of the epoch.
+ * `epoch`: the epoch number.
+ * `objective`: the exact or approximate objective at the end of the previous
+   epoch.
 
 If the callback returns `true`, the optimization will be terminated.
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `size_t` | **`epoch`** | The index of the current epoch. |
-| `double` | **`objective`** | Objective value of the current point. |
-
 ### EndEpoch
 
-Called at the end of a pass over the data. The objective may be exact or
-an estimate depending on `exactObjective` value.
+Called at the end of a pass over the data, for
+[separable functions](#separable-functions). The objective may be exact or an
+estimate depending on the optimizer's `ExactObjective()` value.  Add this
+function to your callback class with your desired implementation:
 
- * `bool EndEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool EndEpoch(OptimizerType& optimizer,
+                FunctionType& function,
+                const MatType& coordinates,
+                const size_t epoch,
+                const double objective);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates at the end of the epoch.
+ * `epoch`: the epoch number.
+ * `objective`: the exact or approximate objective at the end of the epoch.
 
 If the callback returns `true`, the optimization will be terminated.
 
-#### Attributes
+### StepTaken
 
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `size_t` | **`epoch`** | The index of the current epoch. |
-| `double` | **`objective`** | Objective value of the current point. |
+Called after the optimizer has taken any step that modifies the coordinates.
+Add this function to your callback class with your desired implementation:
+
+```c++
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  bool StepTaken(OptimizerType& optimizer,
+                FunctionType& function,
+                MatType& coordinates);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `function`: the function that is being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates after the step (can be modified!).
+
+If the callback returns `true`, the optimization will be terminated.  Note that
+changing the `coordinates` matrix may cause strange behavior for certain
+optimizers---your mileage may vary!
 
 ### GenerationalStepTaken
 
-Called after the evolution of a single generation. Intended specifically for
-MultiObjective Optimizers.
+Called after the evolution of a single generation, for
+[multi-objective functions](#multi-objective-functions).  Add this function to
+your callback class with your desired implementation:
 
- * `bool GenerationalStepTaken(`_`optimizer, function, coordinates, objectives, frontIndices`_`)`
+```c++
+  template<typename OptimizerType,
+           typename... FunctionTypes,
+           typename MatType,
+           typename ObjectivesVecType,
+           typename IndicesType>
+  bool GenerationalStepTaken(OptimizerType& optimizer,
+                             std::tuple<FunctionTypes...>& functions,
+                             MatType& coordinates,
+                             ObjectivesVecType& objectives,
+                             IndicesType& frontIndices);
+```
+
+ * `optimizer`: the actual object on which `Optimize()` was called.
+ * `functions`: the functions that are being optimized (e.g. the first argument
+   given to `optimizer.Optimize()`.
+ * `coordinates`: the coordinates after taking the step.
+ * `objectives`: a vector of column vectors indicating the objective for each
+   element in the population on each objective function.
+ * `frontIndices`: indices of the population that are on the Pareto front.
 
 If the callback returns `true`, the optimization will be terminated.
 
-#### Attributes
-
-| **type** | **name** | **description** |
-|----------|----------|-----------------|
-| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
-| `FunctionType` | **`function`** | The function to be optimized. |
-| `MatType` | **`coordinates`** | The current function parameter. |
-| `ObjectivesVecType` | **`objectives`** | The set of calculated objectives so far. |
-| `IndicesType` | **`frontIndices`** | The indices of the members belonging to Pareto Front. |
-
-## Custom Callbacks
+## Custom Callback Examples
 
 ### Learning rate scheduling
 

doc/function_types.md

@@ -130,8 +130,13 @@ Each of the implemented methods is allowed to have additional cv-modifiers
 The following optimizers can be used with differentiable functions:
 
  * [L-BFGS](#l-bfgs) (`ens::L_BFGS`)
+ * [Forward-backward splitting (FBS)](#forward-backward-splitting-fbs) (`ens::FBS`)
+ * [Fast Iterative Shrinkage-Thresholding Algorithm (FISTA)](#fast-iterative-shrinkage-thresholding-algorithm-fista) (`ens::FISTA`)
+ * [Fast Adaptive Shrinkage/Thresholding Algorithm (FASTA)](#fast-adaptive-shrinkage-thresholding-algorithm-fasta) (`ens::FASTA`)
  * [FrankWolfe](#frank-wolfe) (`ens::FrankWolfe`)
  * [GradientDescent](#gradient-descent) (`ens::GradientDescent`)
+ * [DeltaBarDelta](#deltabardelta) (`ens::DeltaBarDelta`)
+ * [MomentumDeltaBarDelta](#momentum-deltabardelta) (`ens::MomentumDeltaBarDelta`)
  - Any optimizer for [arbitrary functions](#arbitrary-functions)
 
 Each of these optimizers has an `Optimize()` function that is called as
@@ -210,6 +215,12 @@ class LinearRegressionEWGFunction
     g = -2 * data * v;
     return arma::accu(v % v); // equivalent to \| v \|^2
   }
+
+ private:
+  // The data.
+  const arma::mat& data;
+  // The responses to each data point.
+  const arma::rowvec& responses;
 };
 
 int main()
@@ -249,7 +260,7 @@ int main()
   const double time1 = clock.toc();
 
   std::cout << "LinearRegressionFunction with Evaluate() and Gradient() took "
-    << time1 << " seconds to converge to the model: " << std::endl;
+      << time1 << " seconds to converge to the model: " << std::endl;
   std::cout << lrf1Params.t();
 
   // Create the second objective function, which uses EvaluateWithGradient().
@@ -273,6 +284,150 @@ int main()
 
 </details>
 
+### Proximal operators and non-differentiable functions
+
+Some optimization problems involve non-differentiable components, and can be
+expressed as the optimization below:
+
+$$ \operatorname{argmin}_x h(x) = \operatorname{argmin}_x f(x) + g(x). $$
+
+Here, `f(x)` is a regular differentiable function (as in the previous
+subsection), and `g(x)` is a non-differentiable arbitrary function.  These
+classes of functions can still be optimized using *proximal gradient
+optimizers*.  The following proximal gradient optimizers are implemented in
+ensmallen (these can also optimize differentiable functions only, taking `g(x) =
+0`):
+
+ * [Forward-backward splitting (FBS)](#forward-backward-splitting-fbs) (`ens::FBS`)
+ * [Fast Iterative Shrinkage-Thresholding Algorithm (FISTA)](#fast-iteartive-shrinkage-thresholding-algorithm-fista) (`ens::FISTA`)
+ * [Fast Adaptive Shrinkage/Thresholding Algorithm (FASTA)](#fast-adaptive-shrinkage-thresholding-algorithm-fasta) (`ens::FASTA`)
+
+ensmallen implements a few `g(x)` options that can be used with proximal
+gradient optimizers:
+
+ * `L1Penalty(`_`lambda`_`)`: $g(x) = \lambda \| x \|_1$
+ * `L1Constraint(`_`lambda`_`)`: $g(x)$ is the constraint $\| x \|_1 \le \lambda$
+
+For example, by pairing `L1Penalty` with the `LinearRegressionFunction` from the
+previous section, we can implement L1-penalized (sparse) linear regression:
+
+<details>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+#include <ensmallen.hpp>
+
+// Define a differentiable objective function by implementing only
+// EvaluateWithGradient().
+class LinearRegressionEWGFunction
+{
+ public:
+  // Construct the object with the given data matrix and responses.
+  LinearRegressionEWGFunction(const arma::mat& dataIn,
+                              const arma::rowvec& responsesIn) :
+      data(dataIn), responses(responsesIn) { }
+
+  // Simultaneously compute both the objective function and gradient for model
+  // parameters x.  Note that this is faster than implementing Evaluate() and
+  // Gradient() individually because it caches the computation of
+  // (responses - x.t() * data)!
+  double EvaluateWithGradient(const arma::mat& x, arma::mat& g)
+  {
+    const arma::rowvec v = (responses - x.t() * data);
+    g = -2 * data * v;
+    return arma::accu(v % v); // equivalent to \| v \|^2
+  }
+
+ private:
+  // The data.
+  const arma::mat& data;
+  // The responses to each data point.
+  const arma::rowvec& responses;
+};
+
+int main()
+{
+  // First, generate some random data, with 1000 points and 500 dimensions.
+  // This data has no pattern and as such will make a model that's not very
+  // useful---but the purpose here is just demonstration. :)
+  //
+  // For a more "real world" situation, load a dataset from file using X.load()
+  // and y.load() (but make sure the matrix is column-major, so that each
+  // observation/data point corresponds to a *column*, *not* a row.
+  arma::mat data(500, 1000, arma::fill::randn);
+  arma::rowvec responses(1000, arma::fill::randn);
+
+  // Create a starting point for our optimization as the vector of all zeros.
+  // The model has 500 parameters, so the shape is 500x1.
+  arma::mat startingPoint(500, 1, arma::fill::zeros);
+
+  // Construct the objective function f(x), and the penalty function g(x) with a
+  // lambda value of 0.1.
+  LinearRegressionEWGFunction lrf(data, responses);
+  ens::L1Penalty g(0.1);
+
+  // Create the FBS optimizer with default parameters, and optimize the function
+  // f(x) + g(x) (i.e. L1-penalized linear regression).
+  // The ens::FBS class can be replaced with any ensmallen proximal gradient
+  // optimizer.
+  ens::FBS fbs(g);
+  arma::mat lrfParams(startingPoint);
+  fbs.Optimize(lrf, lrfParams);
+
+  // Count the number of nonzeros in the final model.
+  // To get fewer nonzeros, the penalty value (0.1) could be increased.
+  std::cout << "Number of nonzeros in optimized parameter vector: "
+      << arma::accu(lrfParams != 0) << "." << std::endl;
+}
+```
+
+</details>
+
+It is possible to implement a custom proximal operator (`g(x)`).  To do so, a
+class with two methods (`Evaluate()` and `BackwardStep()`) must be defined:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+// Compute the value of g(x).
+double Evaluate(const arma::mat& x);
+
+// Perform a backward step (proximal step) on `x`, given that the forward step
+// size was `stepSize`.
+void BackwardStep(arma::mat& x, const double stepSize);
+```
+
+</details>
+
+A simple implementation is below for the `L1Penalty` class:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+double L1Penalty::Evaluate(const arma::mat& coordinates) const
+{
+  // Compute the L1 penalty.
+  return norm(vectorise(coordinates), 1) * lambda;
+}
+
+void L1Penalty::ProximalStep(arma::mat& coordinates,
+                             const double stepSize) const
+{
+  // Apply the backwards step coordinate-wise.
+  // (See Goldstein, Studer, and Baraniuk 2009, eq. (12).)
+  coordinates.transform([this, stepSize](double val) { return (val > 0.0) ?
+      (std::max(0.0, val - lambda * stepSize)) :
+      (std::min(0.0, val + lambda * stepSize)); });
+}
+```
+
+</details>
+
 ### Partially differentiable functions
 
 Some differentiable functions have the additional property that the gradient
@@ -880,9 +1035,9 @@ arma::mat coordinates(6, 1, arma::fill::randu);
 NSGA2 nsga;
 double bestFrontSum = nsga.Optimize(objectives, coordinates);
 
-// Set `bestFront` to contain all of the coordinates on the best front.
-arma::cube bestFront = optimizer.ParetoFront();
-}
+// If the entire Pareto front is desired, pass it to the Optimize() function:
+arma::cube front, paretoSet;
+double bestFrontSum2 = nsga.Optimize(objectives, coordinates, front, paretoSet);
 ```
 
 </details>
@@ -967,9 +1122,14 @@ double igdPlus = IGDPlus::Evaluate(front, referenceFront);
 ```
 </details>
 
-*Note*: all multi-objective function optimizers have both the function `Optimize()` to find the
-best front, and also the function `ParetoFront()` to return all sets of solutions that are on the
-front.
+*Note*: all multi-objective function optimizers have two versions of
+`Optimize()`: one that finds only the best front, and one that also allows
+passing `arma::cube`s (or similar) to return the Pareto set (e.g. the Pareto
+optimal points in variable space) as well as the entire Pareto front (e.g. all
+sets of solutions on the front):
+
+ * `Optimize(`_`functions`_`,`_`coordinates`_`)`
+ * `Optimize(`_`functions`_`,`_`coordinates`_`,`_`paretoSet`_`,`_`paretoFront`_`)`
 
 The following optimizers can be used with multi-objective functions:
 - [NSGA2](#nsga2)

doc/optimizers.md

@@ -5,9 +5,9 @@
 Active CMA-ES is a variant of the stochastic search algorithm
 CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
 Active CMA-ES actively reduces the uncertainty in unfavourable directions by
-exploiting the information about bad mutations in the covariance matrix 
-update step. This isn't for the purpose of accelerating progress, but 
-instead for speeding up the adaptation of the covariance matrix (which, in 
+exploiting the information about bad mutations in the covariance matrix
+update step. This isn't for the purpose of accelerating progress, but
+instead for speeding up the adaptation of the covariance matrix (which, in
 turn, will lead to faster progress).
 
 #### Constructors
@@ -22,10 +22,10 @@ The _`SelectionPolicyType`_ template parameter refers to the strategy used to
 compute the (approximate) objective function.  The `FullSelection` and
 `RandomSelection` classes are available for use; custom behavior can be achieved
 by implementing a class with the same method signatures.
-The _`TransformationPolicyType`_  template parameter refers to transformation 
-strategy used to map decision variables to the desired domain during fitness 
-evaluation and optimization termination. The `EmptyTransformation` and 
-`BoundaryBoxConstraint` classes are available for use; custom behavior can be 
+The _`TransformationPolicyType`_  template parameter refers to transformation
+strategy used to map decision variables to the desired domain during fitness
+evaluation and optimization termination. The `EmptyTransformation` and
+`BoundaryBoxConstraint` classes are available for use; custom behavior can be
 achieved by implementing a class with the same method signatures.
 
 For convenience the following types can be used:
@@ -55,11 +55,11 @@ the option is not relevant when the `ActiveCMAES<>` optimizer type is being used
 `RandomSelection` policy has the constructor `RandomSelection(`_`fraction`_`)`
 where _`fraction`_ specifies the percentage of separable functions to use to
 estimate the objective function.
-The `transformationPolicy` attribute allows an instantiated 
-`TransformationPolicyType` to be given. The `EmptyTransformation<`_`MatType`_`>` 
+The `transformationPolicy` attribute allows an instantiated
+`TransformationPolicyType` to be given. The `EmptyTransformation<`_`MatType`_`>`
 has no need to be instantiated. `BoundaryBoxConstraint<`_`MatType`_`>` policy has
 the constructor `BoundaryBoxConstraint(`_`lowerBound, upperBound`_`)`
-where  _`lowerBound`_ and _`lowerBound`_ are the lower bound and upper bound of 
+where  _`lowerBound`_ and _`lowerBound`_ are the lower bound and upper bound of
 the coordinates respectively.
 
 #### Examples:
@@ -293,7 +293,7 @@ optimizer.Optimize(f, coordinates);
  * [AdaGrad](#adagrad)
  * [Differentiable separable functions](#differentiable-separable-functions)
 
-## Adagrad
+## AdaGrad
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
 
@@ -603,9 +603,8 @@ arma::mat coords = SCH.GetInitialPoint();
 std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
 
 // obj will contain the minimum sum of objectiveA and objectiveB found on the best front.
-double obj = opt.Optimize(objectives, coords);
-// Now obtain the best front.
-arma::cube bestFront = opt.ParetoFront();
+arma::cube bestSet, bestFront;
+double obj = opt.Optimize(objectives, coords, bestSet, bestFront);
 ```
 
 </details>
@@ -615,26 +614,26 @@ arma::cube bestFront = opt.ParetoFront();
 </summary>
 
 ```c++
-ZDT3<> ZDT_THREE(300);
+ZDT3<> zdt3(300);
 const double lowerBound = 0;
 const double upperBound = 1;
 
 AGEMOEA opt(50, 500, 0.8, 20, 1e-6, 20, lowerBound, upperBound);
-typedef decltype(ZDT_THREE.objectiveF1) ObjectiveTypeA;
-typedef decltype(ZDT_THREE.objectiveF2) ObjectiveTypeB;
+typedef decltype(zdt3.objectiveF1) ObjectiveTypeA;
+typedef decltype(zdt3.objectiveF2) ObjectiveTypeB;
 bool success = true;
-arma::mat coords = ZDT_THREE.GetInitialPoint();
-std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = ZDT_THREE.GetObjectives();
-opt.Optimize(objectives, coords);
-const arma::cube bestFront = opt.ParetoFront();
-  
+arma::mat coords = zdt3.GetInitialPoint();
+std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = zdt3.GetObjectives();
+arma::cube bestSet, bestFront;
+opt.Optimize(objectives, coords, bestSet, bestFront);
+
 NSGA2 opt2(50, 5000, 0.5, 0.5, 1e-3, 1e-6, lowerBound, upperBound);
 // obj2 will contain the minimum sum of objectiveA and objectiveB found on the best front.
-double obj2 = opt2.Optimize(objectives, coords);
- 
-arma::cube NSGAFront = opt2.ParetoFront();
+arma::cube paretoSet, paretoFront;
+double obj2 = opt2.Optimize(objectives, coords, paretoSet, paretoFront);
+
 // Get the IGD score for NSGA front using AGEMOEA as reference.
-double igd = IGD::Evaluate(NSGAFront, bestFront, 1);
+double igd = IGD::Evaluate(paretoFront, bestFront, 1);
 std::cout << igd << std::endl;
 ```
 
@@ -792,6 +791,10 @@ optimizer uses [L-BFGS](#l-bfgs).
 #### Constructors
 
  * `AugLagrangian(`_`maxIterations, penaltyThresholdFactor, sigmaUpdateFactor`_`)`
+ * `AugLagrangianType<_VecType_>(`_`maxIterations, penaltyThresholdFactor, sigmaUpdateFactor`_`)`
+
+When optimizing matrix types other than `arma::mat`, specify `VecType` as the
+corresponding vector type (e.g. `arma::vec` or `coot::fvec`).
 
 #### Attributes
 
@@ -805,46 +808,20 @@ optimizer uses [L-BFGS](#l-bfgs).
 The attributes of the optimizer may also be modified via the member methods
 `MaxIterations()`, `PenaltyThresholdFactor()`, `SigmaUpdateFactor()` and `LBFGS()`.
 
-<details open>
-<summary>Click to collapse/expand example code.
-</summary>
+The `AugLagrangian` optimizer also allows manually specifying the initial
+Lagrange multipliers (`lambda`) and penalty parameter (`sigma`) directly in the
+call to `Optimize()`.  For this, the following version of `Optimize()` should be
+used:
 
-```c++
-/**
- * Optimize the function.  The value '1' is used for the initial value of each
- * Lagrange multiplier.  To set the Lagrange multipliers yourself, use the
- * other overload of Optimize().
- *
- * @tparam LagrangianFunctionType Function which can be optimized by this
- *     class.
- * @param function The function to optimize.
- * @param coordinates Output matrix to store the optimized coordinates in.
- */
-template<typename LagrangianFunctionType>
-bool Optimize(LagrangianFunctionType& function,
-              arma::mat& coordinates);
+ * `opt.Optimize(`_`function, coordinates, lambda, sigma, callbacks...`_`)`
 
-/**
- * Optimize the function, giving initial estimates for the Lagrange
- * multipliers.  The vector of Lagrange multipliers will be modified to
- * contain the Lagrange multipliers of the final solution (if one is found).
- *
- * @tparam LagrangianFunctionType Function which can be optimized by this
- *      class.
- * @param function The function to optimize.
- * @param coordinates Output matrix to store the optimized coordinates in.
- * @param initLambda Vector of initial Lagrange multipliers.  Should have
- *     length equal to the number of constraints.
- * @param initSigma Initial penalty parameter.
- */
-template<typename LagrangianFunctionType>
-bool Optimize(LagrangianFunctionType& function,
-              arma::mat& coordinates,
-              const arma::vec& initLambda,
-              const double initSigma);
-```
+In that call, `lambda` should be a column vector of the same type as
+`coordinates`, and `sigma` is a `double`.  `lambda` and `sigma` will be
+overwritten with the final values of the Lagrange multipliers and penalty
+parameters.
 
-</details>
+If `lambda` and `sigma` are not specified, then 0 is used as the initial value
+for all Lagrange multipliers and 10 is used as the initial penalty parameter.
 
 #### Examples
 
@@ -1112,10 +1089,10 @@ The _`SelectionPolicyType`_ template parameter refers to the strategy used to
 compute the (approximate) objective function.  The `FullSelection` and
 `RandomSelection` classes are available for use; custom behavior can be achieved
 by implementing a class with the same method signatures.
-The _`TransformationPolicyType`_  template parameter refers to transformation 
-strategy used to map decision variables to the desired domain during fitness 
-evaluation and optimization termination. The `EmptyTransformation` and 
-`BoundaryBoxConstraint` classes are available for use; custom behavior can be 
+The _`TransformationPolicyType`_  template parameter refers to transformation
+strategy used to map decision variables to the desired domain during fitness
+evaluation and optimization termination. The `EmptyTransformation` and
+`BoundaryBoxConstraint` classes are available for use; custom behavior can be
 achieved by implementing a class with the same method signatures.
 
 For convenience the following types can be used:
@@ -1145,11 +1122,11 @@ the option is not relevant when the `CMAES<>` optimizer type is being used; the
 `RandomSelection` policy has the constructor `RandomSelection(`_`fraction`_`)`
 where _`fraction`_ specifies the percentage of separable functions to use to
 estimate the objective function.
-The `transformationPolicy` attribute allows an instantiated 
-`TransformationPolicyType` to be given. The `EmptyTransformation<`_`MatType`_`>` 
+The `transformationPolicy` attribute allows an instantiated
+`TransformationPolicyType` to be given. The `EmptyTransformation<`_`MatType`_`>`
 has no need to be instantiated. `BoundaryBoxConstraint<`_`MatType`_`>` policy has
 the constructor `BoundaryBoxConstraint(`_`lowerBound, upperBound`_`)`
-where  _`lowerBound`_ and _`lowerBound`_ are the lower bound and upper bound of 
+where  _`lowerBound`_ and _`lowerBound`_ are the lower bound and upper bound of
 the coordinates respectively.
 
 #### Examples:
@@ -1282,6 +1259,70 @@ optimizer.Optimize(f, coordinates);
  * [Differential Evolution in Wikipedia](https://en.wikipedia.org/wiki/Differential_Evolution)
  * [Arbitrary functions](#arbitrary-functions)
 
+## DeltaBarDelta
+
+*An optimizer for [differentiable functions](#differentiable-functions).*
+
+A Gradient Descent variant that adapts learning rates for each parameter to improve convergence. If the current gradient and the exponential average of past gradients corresponding to a parameter have the same sign, then the step size for that parameter is incremented by `kappa`. Otherwise, it is decreased by a proportion `phi` of its current value (additive increase, multiplicative decrease).
+
+***Notes:***
+
+ - DeltaBarDelta is very sensitive to its parameters (`kappa` and `phi`) hence a good
+   hyperparameter selection is necessary as its default may not fit every case.
+   Typically, `kappa` should be smaller than the step size.
+
+ - This implementation uses a minStepSize parameter to set a lower bound for the learning
+   rate. This prevents the learning rate from dropping to zero, which can occur due to
+   floating-point underflow. For tasks which require extreme fine-tuning, you may need to
+   lower this parameter below its default value (1e-8) in order to allow for smaller
+   learning rates.
+
+#### Constructors
+
+ * `DeltaBarDelta()`
+ * `DeltaBarDelta(`_`stepSize`_`)`
+ * `DeltaBarDelta(`_`stepSize, maxIterations, tolerance`_`)`
+ * `DeltaBarDelta(`_`stepSize, maxIterations, tolerance, kappa, phi, theta, minStepSize, resetPolicy`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Initial step size. | `1.0` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`**  | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `double` | **`kappa`** | Additive increase constant for step size when gradient signs persist. | `0.2` |
+| `double` | **`phi`** | Multiplicative decrease factor for step size when gradient signs flip. | `0.2` |
+| `double` | **`theta`** | Decay rate for computing the exponential average of past gradients. | `0.5` |
+| `double` | **`minStepSize`** | Minimum allowed step size for any parameter. | `1e-8` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every `Optimize()` call. | `true` |
+
+Attributes of the optimizer may also be modified via the member methods
+`StepSize()`, `MaxIterations()`, `Tolerance()`, `Kappa()`, `Phi()`, `Theta()`, `MinStepSize()` and `ResetPolicy()`.
+
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+DeltaBarDelta optimizer(0.001, 0, 1e-15, 0.0001, 0.2, 0.8);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [Increased rates of convergence through learning rate adaptation (pdf)](https://www.academia.edu/download/32005051/Jacobs.NN88.pdf)
+ * [Differentiable functions](#differentiable-functions)
+ * [Gradient Descent](#gradient-descent)
+
 ## DemonAdam
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -1294,7 +1335,7 @@ total contribution of a gradient to all future updates.
  * `DemonAdam()`
  * `DemonAdam(`_`stepSize, batchSize`_`)`
  * `DemonAdam(`_`stepSize, batchSize, momentum, beta1, beta2, eps, maxIterations, tolerance, shuffle`_`)`
- * `DemonAdam(`_`stepSize, batchSize, momentum, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `DemonAdam(`_`stepSize, batchSize, momentum, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 Note that the `DemonAdam` class is based on
 the `DemonAdamType<`_`UpdateRule`_`>` class with _`UpdateRule`_` = AdamUpdate`.
@@ -1326,12 +1367,20 @@ For convenience the following typedefs have been defined:
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective at the end of optimization.  (This could be computationally expensive!) | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Momentum()`, `MomentumIterations()`, `Beta1()`,
 `Beta2()`, `Eps()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, and
 `ResetPolicy()`.
 
+***Note:*** if `exactObjective` is `false`, then `Optimize(f, coordinates)` will
+return an estimate of the objective function.  This estimate is the sum of the
+objectives obtained on the last pass of the separable functions.  The estimate
+will not include contributions from any separable functions not visited in the
+last pass (e.g., if `maxIterations` is not an integer multiple of
+`f.NumFunctions()`).
+
 #### Examples
 
 <details open>
@@ -1383,7 +1432,7 @@ optimizer:
  * `DemonSGD()`
  * `DemonSGD(`_`stepSize, batchSize`_`)`
  * `DemonSGD(`_`stepSize, batchSize, momentum, maxIterations, tolerance, shuffle`_`)`
- * `DemonSGD(`_`stepSize, batchSize, momentum, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `DemonSGD(`_`stepSize, batchSize, momentum, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -1396,11 +1445,19 @@ optimizer:
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective at the end of optimization.  (This could be computationally expensive!) | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Momentum()`, `MomentumIterations()`,
 `MaxIterations()`, `Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
 
+***Note:*** if `exactObjective` is `false`, then `Optimize(f, coordinates)` will
+return an estimate of the objective function.  This estimate is the sum of the
+objectives obtained on the last pass of the separable functions.  The estimate
+will not include contributions from any separable functions not visited in the
+last pass (e.g., if `maxIterations` is not an integer multiple of
+`f.NumFunctions()`).
+
 #### Examples
 
 <details open>
@@ -1486,6 +1543,298 @@ optimizer.Optimize(f, coordinates);
  * [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](https://arxiv.org/pdf/1611.01505.pdf)
  * [Differentiable separable functions](#differentiable-separable-functions)
 
+## Fast Adaptive Shrinkage/Thresholding Algorithm (FASTA)
+
+*An optimizer for [differentiable functions](#differentiable-functions) that may
+also include non-differentiable L1 penalties or similar.*
+
+The Fast Adaptive Shrinkage/Thresholding Algorithm (FASTA) is a proximal
+gradient optimization technique to optimize composite functions of the form
+
+```
+h(x) = f(x) + g(x).
+```
+
+Here, `f(x)` is a differentiable function, and `g(x)` is an arbitrary
+non-differentiable function that has a corresponding *proximal operator*.
+In this situation, other ensmallen optimizers for differentiable functions
+cannot be used, since `g(x)` is not differentiable.  To work around this, FISTA
+takes a *forward step* that is a standard gradient descent step on `f(x)`, and
+then a *backward step* that is the proximal operator defined by `g(x)`.
+
+For `FASTA`, the function `f(x)` is defined in the standard ensmallen way (it is
+passed to `Optimize()`), and the function `g(x)` is defined by a template
+parameter.
+
+FASTA differs from [FBS](#forward-backward-splitting-fbs) in that it uses a
+predictive step (similar to momentum) and a line search to choose
+step sizes.  Like
+[FISTA](#fast-iterative-shrinkage-thresholding-algorithm-fista), the maximum
+allowable step size is automatically estimated, unless it is specifically
+provided.
+
+FASTA differs from FISTA in its line search strategy: FASTA uses a non-monotone
+line search, which can allow the objective function to increase between
+iterations.  FASTA also has enhanced convergence criteria as compared to FISTA;
+FASTA uses the residual instead of an absolute tolerance on the objective.
+
+#### Constructors
+
+ * `FASTA()`
+ * `FASTA(`_`maxIterations, tolerance, maxLineSearchSteps, stepSizeAdjustment, lineSearchLookback, estimateStepSize, estimateTrials, maxStepSize`_`)`
+ * `FASTA(`_`backwardStep`_`)`
+ * `FASTA(`_`backwardStep, maxIterations, tolerance, maxLineSearchSteps, stepSizeAdjustment, lineSearchLookback, estimateStepSize, estimateTrials, maxStepSize`_`)`
+
+The _`backwardStep`_ parameter specifies the function `g(x)` to optimize.  A few
+options are readily available:
+
+ * `L1Penalty(`_`lambda`_`)`
+   - This is for the L1 penalty function `g(x) = lambda * || x ||_1`.
+ * `L1Constraint(`_`lambda`_`)`
+   - This is for the hard constraint `|| x ||_1 <= lambda`.
+
+The `FASTA` class takes the penalty type (`L1Penalty` or `L1Constraint`) as its
+first template parameter.  This does not need to be explicitly specified if the
+default is used (`L1Penalty`) or if a constructor form specifying `backwardStep`
+is used.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `10000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance on objective to terminate algorithm. | `1e-10` |
+| `size_t` | **`maxLineSearchSteps`** | Maximum number of line search step attempts to take a step (0 means no limit). | `50` |
+| `double` | **`stepSizeAdjustment`** | Multiplicative amount to shrink or increase step size at each step of the line search. | `2.0` |
+| `size_t` | **`lineSearchLookback`** | Number of previous iterations' objective values to use for relaxed non-monotone line search conditions. | `10` |
+| `bool` | **`estimateStepSize`** | If `true`, `maxStepSize` is computed by estimating the Lipschitz constant of `f(x)`. | `true` |
+| `size_t` | **`estimateTrials`** | Number of random trials to perform to estimate the Lipschitz constant of `f(x)`. | `10` |
+| `double` | **`maxStepSize`** | Maximum allowable step size for any line search.  Ignored (an estimate is used instead) if `estimateStepSize` is `true`. |
+| `double` | **`lambda`** | L1 penalty parameter or constraint parameter, for `L1Penalty` or `L1Constraint` backward step classes. | `0` |
+
+The attributes of the optimizer may also be modified via the member methods
+`MaxIterations()`, `Tolerance()`, `StepSizeAdjustment()`,
+`LineSearchLookback()`, `EstimateStepSize()`, `EstimateTrials()`, and
+`MaxStepSize()`.  The backward step object can be accessed and modified with the
+`BackwardStep()` member method.
+
+If `L1Penalty` or `L1Constraint` is used as the backward step type, the value of
+lambda can be accessed with `BackwardStep().Lambda()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+// f(x) is the Rosenbrock function.
+RosenbrockFunction f;
+// g(x) is the L1 penalty (with lambda = 0.1).
+L1Penalty g(0.1);
+
+arma::mat coordinates = f.GetInitialPoint();
+// FASTA will optimize h(x) = f(x) + g(x),
+// which here is the L1-penalized Rosenbrock function.
+FASTA optimizer(g, 1000, 1e-6);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [Forward-Backward Splitting (FBS)](#forward-backward-splitting-fbs)
+ * [Fast Iterative Shrinkage-Thresholding Algorithm (FISTA)](#fast-iterative-shrinkage-thresholding-algorithm-fista)
+ * [A Field Guide To Forward-Backward Splitting With A FASTA Implementation](https://arxiv.org/pdf/1411.3406.pdf)
+ * [Proximal Operators on Wikipedia](https://en.wikipedia.org/wiki/Proximal_operator)
+
+## Forward-Backward Splitting (FBS)
+
+*An optimizer for [differentiable functions](#differentiable-functions) that may
+also include non-differentiable L1 penalties or similar.*
+
+Forward-backward splitting (FBS) is a proximal gradient optimization technique
+to optimize composite functions of the form
+
+```
+h(x) = f(x) + g(x).
+```
+
+Here, `f(x)` is a differentiable function, and `g(x)` is an arbitrary
+non-differentiable function that has a corresponding *proximal operator*.
+In this situation, other ensmallen optimizers for differentiable functions
+cannot be used, since `g(x)` is not differentiable.  To work around this, FBS
+takes a *forward step* that is a standard gradient descent step on `f(x)`, and
+then a *backward step* that is the proximal operator defined by `g(x)`.
+
+For `FBS`, the function `f(x)` is defined in the standard ensmallen way (it is
+passed to `Optimize()`), and the function `g(x)` is defined by a template
+parameter.
+
+#### Constructors
+
+ * `FBS()`
+ * `FBS(`_`stepSize, maxIterations, tolerance`_`)`
+ * `FBS(`_`backwardStep`_`)`
+ * `FBS(`_`backwardStep, stepSize, maxIterations, tolerance`_`)`
+
+The _`backwardStep`_ parameter specifies the function `g(x)` to optimize.  A few
+options are readily available:
+
+ * `L1Penalty(`_`lambda`_`)`
+   - This is for the L1 penalty function `g(x) = lambda * || x ||_1`.
+ * `L1Constraint(`_`lambda`_`)`
+   - This is for the hard constraint `|| x ||_1 <= lambda`.
+
+The `FBS` class takes the penalty type (`L1Penalty` or `L1Constraint`) as its
+first template parameter.  This does not need to be explicitly specified if the
+default is used (`L1Penalty`) or if a constructor form specifying `backwardStep`
+is used.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `10000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance objective to terminate algorithm. | `1e-10` |
+| `double` | **`lambda`** | L1 penalty parameter or constraint parameter, for `L1Penalty` or `L1Constraint` backward step classes. | `0` |
+
+The attributes of the optimizer may also be modified via the member methods
+`StepSize()`, `MaxIterations()`, and `Tolerance()`.  The backward step object
+can be accessed and modified with the `BackwardStep()` member method.
+
+If `L1Penalty` or `L1Constraint` is used as the backward step type, the value of
+lambda can be accessed with `BackwardStep().Lambda()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+// f(x) is the Rosenbrock function.
+RosenbrockFunction f;
+// g(x) is the L1 penalty (with lambda = 0.1).
+L1Penalty g(0.1);
+
+arma::mat coordinates = f.GetInitialPoint();
+// FBS will optimize h(x) = f(x) + g(x),
+// which here is the L1-penalized Rosenbrock function.
+FBS optimizer(g, 0.001, 1000, 1e-5);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [Fast Iterative Shrinkage-Thresholding Algorithm (FISTA)](#fast-iterative-shrinkage-thresholding-algorithm-fista)
+ * [Fast Adaptive Shrinkage/Thresholding Algorithm (FASTA)](#fast-adaptive-shrinkage-thresholding-algorithm-fasta) (`ens::FASTA`)
+ * [A Field Guide To Forward-Backward Splitting With A FASTA Implementation](https://arxiv.org/pdf/1411.3406.pdf)
+ * [Proximal Operators on Wikipedia](https://en.wikipedia.org/wiki/Proximal_operator)
+
+## Fast Iterative Shrinkage-Thresholding Algorithm (FISTA)
+
+*An optimizer for [differentiable functions](#differentiable-functions) that may
+also include non-differentiable L1 penalties or similar.*
+
+The Fast Iterative Shrinkage-Thresholding Algorithm (FISTA) is a proximal
+gradient optimization technique to optimize composite functions of the form
+
+```
+h(x) = f(x) + g(x).
+```
+
+Here, `f(x)` is a differentiable function, and `g(x)` is an arbitrary
+non-differentiable function that has a corresponding *proximal operator*.
+In this situation, other ensmallen optimizers for differentiable functions
+cannot be used, since `g(x)` is not differentiable.  To work around this, FISTA
+takes a *forward step* that is a standard gradient descent step on `f(x)`, and
+then a *backward step* that is the proximal operator defined by `g(x)`.
+
+For `FISTA`, the function `f(x)` is defined in the standard ensmallen way (it is
+passed to `Optimize()`), and the function `g(x)` is defined by a template
+parameter.
+
+FISTA differs from [FBS](#forward-backward-splitting-fbs) in that it uses a
+predictive step (similar to momentum) and a line search to choose step size.
+The maximum allowable step size is automatically estimated, unless it is
+specifically provided.
+
+#### Constructors
+
+ * `FISTA()`
+ * `FISTA(`_`maxIterations, tolerance, maxLineSearchSteps, stepSizeAdjustment, estimateStepSize, estimateTrials, maxStepSize`_`)`
+ * `FISTA(`_`backwardStep`_`)`
+ * `FISTA(`_`backwardStep, maxIterations, tolerance, maxLineSearchSteps, stepSizeAdjustment, estimateStepSize, estimateTrials, maxStepSize`_`)`
+
+The _`backwardStep`_ parameter specifies the function `g(x)` to optimize.  A few
+options are readily available:
+
+ * `L1Penalty(`_`lambda`_`)`
+   - This is for the L1 penalty function `g(x) = lambda * || x ||_1`.
+ * `L1Constraint(`_`lambda`_`)`
+   - This is for the hard constraint `|| x ||_1 <= lambda`.
+
+The `FISTA` class takes the penalty type (`L1Penalty` or `L1Constraint`) as its
+first template parameter.  This does not need to be explicitly specified if the
+default is used (`L1Penalty`) or if a constructor form specifying `backwardStep`
+is used.
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `10000` |
+| `double` | **`tolerance`** | Maximum absolute tolerance on objective to terminate algorithm. | `1e-10` |
+| `size_t` | **`maxLineSearchSteps`** | Maximum number of line search step attempts to take a step (0 means no limit). | `50` |
+| `double` | **`stepSizeAdjustment`** | Multiplicative amount to shrink or increase step size at each step of the line search. | `2.0` |
+| `bool` | **`estimateStepSize`** | If `true`, `maxStepSize` is computed by estimating the Lipschitz constant of `f(x)`. | `true` |
+| `size_t` | **`estimateTrials`** | Number of random trials to perform to estimate the Lipschitz constant of `f(x)`. | `10` |
+| `double` | **`maxStepSize`** | Maximum allowable step size for any line search.  Ignored (an estimate is used instead) if `estimateStepSize` is `true`. |
+| `double` | **`lambda`** | L1 penalty parameter or constraint parameter, for `L1Penalty` or `L1Constraint` backward step classes. | `0` |
+
+The attributes of the optimizer may also be modified via the member methods
+`MaxIterations()`, `Tolerance()`, `StepSizeAdjustment()`, `EstimateStepSize()`,
+`EstimateTrials()`, and `MaxStepSize()`.  The backward step object can be
+accessed and modified with the `BackwardStep()` member method.
+
+If `L1Penalty` or `L1Constraint` is used as the backward step type, the value of
+lambda can be accessed with `BackwardStep().Lambda()`.
+
+#### Examples
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+// f(x) is the Rosenbrock function.
+RosenbrockFunction f;
+// g(x) is the L1 penalty (with lambda = 0.1).
+L1Penalty g(0.1);
+
+arma::mat coordinates = f.GetInitialPoint();
+// FISTA will optimize h(x) = f(x) + g(x),
+// which here is the L1-penalized Rosenbrock function.
+// The maximum step size will be automatically estimated.
+FISTA optimizer(g, 1000, 1e-8);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+
+ * [Forward-Backward Splitting (FBS)](#forward-backward-splitting-fbs)
+ * [Fast Adaptive Shrinkage/Thresholding Algorithm (FASTA)](#fast-adaptive-shrinkage-thresholding-algorithm-fasta) (`ens::FASTA`)
+ * [A Fast Iterative Shrinkage-Thresholding Algorithm for Linear Inverse Problems](https://www.ceremade.dauphine.fr/~carlier/FISTA)
+ * [A Field Guide To Forward-Backward Splitting With A FASTA Implementation](https://arxiv.org/pdf/1411.3406.pdf)
+ * [Proximal Operators on Wikipedia](https://en.wikipedia.org/wiki/Proximal_operator)
+
 ## Frank-Wolfe
 
 *An optimizer for [differentiable functions](#differentiable-functions) that may also be constrained.*
@@ -1498,11 +1847,12 @@ Frank-Wolfe is a technique to minimize a continuously differentiable convex func
  * `FrankWolfe<`_`LinearConstrSolverType, UpdateRuleType`_`>(`_`linearConstrSolver, updateRule, maxIterations, tolerance`_`)`
 
 The _`LinearConstrSolverType`_ template parameter specifies the constraint
-domain D for the problem.  The `ConstrLpBallSolver` and
-`ConstrStructGroupSolver<GroupLpBall>` classes are available for use; the former
-restricts D to the unit ball of the specified l-p norm.  Other constraint types
-may be implemented as a class with the same method signatures as either of the
-existing classes.
+domain D for the problem.  The `ConstrLpBallSolver` (itself a class template,
+`ConstrLpBallSolver<T>`, change `T` if a different matrix type is required)
+and `ConstrStructGroupSolver<GroupLpBall>` classes are available for use; the
+former restricts D to the unit ball of the specified l-p norm.  Other constraint
+types may be implemented as a class with the same method signatures as either of
+the existing classes.
 
 The _`UpdateRuleType`_ template parameter specifies the update rule used by the
 optimizer.  The `UpdateClassic` and `UpdateLineSearch` classes are available for
@@ -1528,10 +1878,6 @@ For convenience the following typedefs have been defined:
 Attributes of the optimizer may also be changed via the member methods
 `LinearConstrSolver()`, `UpdateRule()`, `MaxIterations()`, and `Tolerance()`.
 
-#### Examples:
-
-TODO
-
 #### See also:
 
  * [An algorithm for quadratic programming](https://pdfs.semanticscholar.org/3a24/54478a94f1e66a3fc5d209e69217087acbc0.pdf)
@@ -1610,11 +1956,17 @@ Gradient Descent is a technique to minimize a function. To find a local minimum
 of a function using gradient descent, one takes steps proportional to the
 negative of the gradient of the function at the current point.
 
+Note that Gradient Descent is an extremely simple optimizer. For more advanced, adaptive optimizers, consider DeltaBarDelta, MomentumDeltaBarDelta, or established stochastic variants such as Adam, RMSProp, and AdaGrad.
+
 #### Constructors
 
  * `GradientDescent()`
  * `GradientDescent(`_`stepSize`_`)`
  * `GradientDescent(`_`stepSize, maxIterations, tolerance`_`)`
+ * `GradientDescent(`_`stepSize, maxIterations, tolerance, updatePolicy, decayPolicy, resetPolicy`_`)`
+
+Note that `GradientDescent` is based on the templated type
+`GradientDescentType<`_`UpdatePolicyType, DecayPolicyType`_`>` with _`UpdatePolicyType`_` = VanillaUpdate` and _`DecayPolicyType`_` = NoDecay`.
 
 #### Attributes
 
@@ -1625,7 +1977,9 @@ negative of the gradient of the function at the current point.
 | `size_t` | **`tolerance`**  | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 
 Attributes of the optimizer may also be changed via the member methods
-`StepSize()`, `MaxIterations()`, and `Tolerance()`.
+`StepSize()`, `MaxIterations()`, `Tolerance()`, `UpdatePolicy()`,
+`DecayPolicy()`, and `ResetPolicy()`.
+
 
 #### Examples:
 
@@ -1745,10 +2099,10 @@ optimizer.Optimize(f, coordinates);
 *An optimizer for [separable functions](#separable-functions).*
 
 IPOP CMA-ES (Increasing Population Size CMA-ES) is an extension of the
-Covariance Matrix Adaptation Evolution Strategy (CMA-ES). It introduces a 
-restart mechanism that progressively increases the population size. This 
+Covariance Matrix Adaptation Evolution Strategy (CMA-ES). It introduces a
+restart mechanism that progressively increases the population size. This
 approach is beneficial for optimizing multi-modal functions,
-characterized by numerous local optima. The restart mechanism is designed to 
+characterized by numerous local optima. The restart mechanism is designed to
 improve the adaptability of CMA-ES by improving the likelihood of escaping
 local optima, thus increasing the chances of discovering the global optimum.
 
@@ -2092,12 +2446,87 @@ The attributes of the LRSDP optimizer may only be accessed via member methods.
 | `size_t` | **`MaxIterations()`** | Maximum number of iterations before termination. | `1000` |
 | `AugLagrangian` | **`AugLag()`** | The internally-held Augmented Lagrangian optimizer. | **n/a** |
 
+Because `LRSDP` uses the [`AugLagrangian`](#auglagrangian) optimizer internally,
+an additional overload of `Optimize()` is supplied to allow specifying the
+initial Lagrange multiplier estimates and penalty parameter:
+
+ * `lrsdp.Optimize(`_`coordinates, lambda, sigma, callbacks...`_`)`
+
+In that call, `lambda` should be a column vector of the same type as
+`coordinates`, and `sigma` is a `double`.  `lambda` and `sigma` will be
+overwritten with the final values of the Lagrange multipliers and penalty
+parameters.
+
+If `lambda` and `sigma` are not specified, then 0 is used as the initial value
+for all Lagrange multipliers and 10 is used as the initial penalty parameter.
+
 #### See also:
 
  * [A Nonlinear Programming Algorithm for Solving Semidefinite Programs via Low-rank Factorization](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.682.1520&rep=rep1&type=pdf)
  * [Semidefinite programming on Wikipedia](https://en.wikipedia.org/wiki/Semidefinite_programming)
  * [Semidefinite programs](#semidefinite-programs) (includes example usage of `PrimalDualSolver`)
 
+## Momentum DeltaBarDelta
+
+*An optimizer for [differentiable functions](#differentiable-functions).*
+
+A [DeltaBarDelta](#deltabardelta) variant that incorporates the following modifications:
+ - In the original DeltaBarDelta, the momentum term (`delta_bar`) is used
+   solely for sign comparison with the current gradient and does not
+   participate in the parameter update. In this modified variant, the
+   momentum term (`velocity`) is directly used to update the parameters.
+ - Instead of adjusting the step size directly, each parameter maintains
+   a gain value initialized to 1.0. Updates apply additive increases or
+   multiplicative decreases to this gain. The effective step size for a
+   parameter is the product of its initial step size and its current gain.
+
+Note: This variant originates from optimization of the t-SNE cost function.
+
+#### Constructors
+
+ * `MomentumDeltaBarDelta()`
+ * `MomentumDeltaBarDelta(`_`stepSize`_`)`
+ * `MomentumDeltaBarDelta(`_`stepSize, maxIterations, tolerance`_`)`
+ * `MomentumDeltaBarDelta(`_`stepSize, maxIterations, tolerance, kappa, phi, momentum, minGain, resetPolicy`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `double` | **`stepSize`** | Initial step size. | `1.0` |
+| `size_t` | **`maxIterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
+| `double` | **`tolerance`**  | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
+| `double` | **`kappa`** | Additive increase constant for step size. | `0.2` |
+| `double` | **`phi`** | Multiplicative decrease factor for step size. | `0.8` |
+| `double` | **`momentum`** | The momentum hyperparameter. | `0.5` |
+| `double` | **`minGain`** | Minimum allowed gain (scaling factor) for any parameter. | `1e-8` |
+| `bool` | **`resetPolicy`** | If true, parameters are reset before every `Optimize()` call. | `true` |
+
+Attributes of the optimizer may also be modified via the member methods
+`StepSize()`, `MaxIterations()`, `Tolerance()`, `Kappa()`, `Phi()`, `Momentum()`, `MinGain()`, and `ResetPolicy()`.
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+MomentumDeltaBarDelta optimizer(0.001, 0, 1e-15, 0.2, 0.8, 0.5);
+optimizer.Optimize(f, coordinates);
+```
+
+</details>
+
+#### See also:
+ * [t-SNE Implementations](https://lvdmaaten.github.io/tsne/)
+ * [Increased rates of convergence through learning rate adaptation (pdf)](https://www.academia.edu/download/32005051/Jacobs.NN88.pdf)
+ * [Differentiable functions](#differentiable-functions)
+ * [Gradient Descent](#gradient-descent)
+
 ## Momentum SGD
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
@@ -2365,15 +2794,19 @@ optimizer.Optimize(f, coordinates);
  * [Differentiable separable functions](#differentiable-separable-functions)
 
 ## MOEA/D-DE
+
 *An optimizer for arbitrary multi-objective functions.*
-MOEA/D-DE (Multi Objective Evolutionary Algorithm based on Decomposition - Differential Evolution) is a multi
-objective optimization algorithm. It works by decomposing the problem into a number of scalar optimization
-subproblems which are solved simultaneously per generation. MOEA/D in itself is a framework, this particular
-algorithm uses Differential Crossover followed by Polynomial Mutation to create offsprings which are then
-decomposed to form a Single Objective Problem. A diversity preserving mechanism is also employed which encourages
-a varied set of solution.
+MOEA/D-DE (Multi Objective Evolutionary Algorithm based on Decomposition -
+Differential Evolution) is a multi objective optimization algorithm. It works by
+decomposing the problem into a number of scalar optimization subproblems which
+are solved simultaneously per generation. MOEA/D in itself is a framework, this
+particular algorithm uses Differential Crossover followed by Polynomial Mutation
+to create offsprings which are then decomposed to form a Single Objective
+Problem. A diversity preserving mechanism is also employed which encourages a
+varied set of solutions.
 
 #### Constructors
+
 * `MOEAD<`_`InitPolicyType, DecompPolicyType`_`>()`
 * `MOEAD<`_`InitPolicyType, DecompPolicyType`_`>(`_`populationSize, maxGenerations, crossoverProb,  neighborProb, neighborSize, distributionIndex, differentialWeight, maxReplace, epsilon, lowerBound, upperBound`_`)`
 
@@ -2443,9 +2876,8 @@ typedef decltype(SCH.objectiveB) ObjectiveTypeB;
 arma::mat coords = SCH.GetInitialPoint();
 std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
 // obj will contain the minimum sum of objectiveA and objectiveB found on the best front.
-double obj = opt.Optimize(objectives, coords);
-// Now obtain the best front.
-arma::cube bestFront = opt.ParetoFront();
+arma::cube paretoSet, paretoFront;
+double obj = opt.Optimize(objectives, coords, paretoSet, paretoFront);
 ```
 </details>
 
@@ -2509,9 +2941,8 @@ arma::mat coords = SCH.GetInitialPoint();
 std::tuple<ObjectiveTypeA, ObjectiveTypeB> objectives = SCH.GetObjectives();
 
 // obj will contain the minimum sum of objectiveA and objectiveB found on the best front.
-double obj = opt.Optimize(objectives, coords);
-// Now obtain the best front.
-arma::cube bestFront = opt.ParetoFront();
+arma::cube paretoSet, paretoFront;
+double obj = opt.Optimize(objectives, coords, paretoSet, paretoFront);
 ```
 
 </details>
@@ -3419,7 +3850,10 @@ Attributes of the optimizer can also be modified via the member methods
 
 The `Snapshots()` function returns a `std::vector<arma::mat>&` (a vector of
 snapshots of the parameters), not a `size_t` representing the maximum number of
-snapshots.
+snapshots.  If a different matrix type or gradient type was specified during the
+optimization, then `Snapshots()` should be called as
+`Snapshots<MatType, GradType>()`; if this is not done, an exception will be
+thrown.
 
 Note that the default value for `updatePolicy` is the default constructor for
 the `UpdatePolicyType`.

include/ensmallen.hpp

@@ -34,7 +34,16 @@
 
 #include <armadillo>
 
-#if ((ARMA_VERSION_MAJOR < 10) || ((ARMA_VERSION_MAJOR == 10) && (ARMA_VERSION_MINOR < 8)))
+#if defined(COOT_VERSION_MAJOR) && \
+    ((COOT_VERSION_MAJOR >= 2) || \
+     (COOT_VERSION_MAJOR == 2 && COOT_VERSION_MINOR >= 1))
+  // The version of Bandicoot is new enough that we can use it.
+  #undef ENS_HAVE_COOT
+  #define ENS_HAVE_COOT
+#endif
+
+#if ((ARMA_VERSION_MAJOR < 10) || \
+    ((ARMA_VERSION_MAJOR == 10) && (ARMA_VERSION_MINOR < 8)))
   #error "need Armadillo version 10.8 or newer"
 #endif
 
@@ -69,7 +78,10 @@
 #include "ensmallen_bits/log.hpp" // TODO: should move to another place
 
 #include "ensmallen_bits/utility/any.hpp"
-#include "ensmallen_bits/utility/arma_traits.hpp"
+#include "ensmallen_bits/utility/proxies.hpp"
+#include "ensmallen_bits/utility/function_traits.hpp"
+#include "ensmallen_bits/utility/using.hpp"
+#include "ensmallen_bits/utility/detect_callbacks.hpp"
 #include "ensmallen_bits/utility/indicators/epsilon.hpp"
 #include "ensmallen_bits/utility/indicators/igd.hpp"
 #include "ensmallen_bits/utility/indicators/igd_plus.hpp"
@@ -108,9 +120,13 @@
 #include "ensmallen_bits/cd/cd.hpp"
 #include "ensmallen_bits/cne/cne.hpp"
 #include "ensmallen_bits/de/de.hpp"
+#include "ensmallen_bits/delta_bar_delta/delta_bar_delta.hpp"
+#include "ensmallen_bits/delta_bar_delta/momentum_delta_bar_delta.hpp"
 #include "ensmallen_bits/eve/eve.hpp"
+#include "ensmallen_bits/fasta/fasta.hpp"
+#include "ensmallen_bits/fbs/fbs.hpp"
+#include "ensmallen_bits/fista/fista.hpp"
 #include "ensmallen_bits/ftml/ftml.hpp"
-
 #include "ensmallen_bits/fw/frank_wolfe.hpp"
 #include "ensmallen_bits/gradient_descent/gradient_descent.hpp"
 #include "ensmallen_bits/grid_search/grid_search.hpp"

include/ensmallen_bits/ada_belief/ada_belief.hpp

@@ -97,7 +97,7 @@ class AdaBelief
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ada_belief/ada_belief_update.hpp

@@ -79,6 +79,8 @@ class AdaBeliefUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -89,10 +91,16 @@ class AdaBeliefUpdate
      */
     Policy(AdaBeliefUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        epsilon(ElemType(parent.epsilon)),
         iteration(0)
     {
       m.zeros(rows, cols);
       s.zeros(rows, cols);
+      // Prevent underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -109,18 +117,18 @@ class AdaBeliefUpdate
       // Increment the iteration counter variable.
       ++iteration;
 
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      s *= parent.beta2;
-      s += (1 - parent.beta2) * arma::pow(gradient - m, 2.0) + parent.epsilon;
+      s *= beta2;
+      s += (1 - beta2) * pow(gradient - m, 2) + epsilon;
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
       // And update the iterate.
-      iterate -= ((m / biasCorrection1) * stepSize) / (arma::sqrt(s /
-          biasCorrection2) + parent.epsilon);
+      iterate -= ((m / biasCorrection1) * ElemType(stepSize)) /
+          (sqrt(s / biasCorrection2) + epsilon);
     }
 
    private:
@@ -133,6 +141,11 @@ class AdaBeliefUpdate
     // The exponential moving average of squared gradient values.
     GradType s;
 
+    // Parent parameters converted to the element type of the matrix.
+    ElemType beta1;
+    ElemType beta2;
+    ElemType epsilon;
+
     // The number of iterations.
     size_t iteration;
   };

include/ensmallen_bits/ada_bound/ada_bound.hpp

@@ -107,7 +107,7 @@ class AdaBoundType
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(DecomposableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ada_bound/ada_bound_update.hpp

@@ -96,6 +96,8 @@ class AdaBoundUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -105,10 +107,24 @@ class AdaBoundUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdaBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0), iteration(0)
+        parent(parent),
+        finalLr(ElemType(parent.finalLr)),
+        gamma(ElemType(parent.gamma)),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        first(true),
+        initialStepSize(0),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
+
+      // Check for underflows in conversions.
+      if (gamma == ElemType(0) && parent.gamma != 0.0)
+        gamma = 10 * std::numeric_limits<ElemType>::epsilon();
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -129,30 +145,30 @@ class AdaBoundUpdate
       if (first)
       {
         first = false;
-        initialStepSize = stepSize;
+        initialStepSize = ElemType(stepSize);
       }
 
       // Increment the iteration counter variable.
       ++iteration;
 
       // Decay the first and second moment running average coefficient.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
-      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma * iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma * iteration));
+      const ElemType fl = finalLr * ElemType(stepSize) / initialStepSize;
+      const ElemType lower = fl * (1 - 1 / (gamma * iteration + 1));
+      const ElemType upper = fl * (1 + 1 / (gamma * iteration));
 
-       // Applies bounds on actual learning rate.
-      iterate -= arma::clamp((stepSize *
-          std::sqrt(biasCorrection2) / biasCorrection1) / (arma::sqrt(v) +
-          parent.epsilon), lower, upper) % m;
+      // Applies bounds on actual learning rate.
+      iterate -= clamp((ElemType(stepSize) *
+          std::sqrt(biasCorrection2) / biasCorrection1) / (sqrt(v) + epsilon),
+          lower, upper) % m;
     }
 
    private:
@@ -165,11 +181,18 @@ class AdaBoundUpdate
     // The exponential moving average of squared gradient values.
     GradType v;
 
+    // Parameters of the parent, casted to the element type of the problem.
+    ElemType finalLr;
+    ElemType gamma;
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // Whether this is the first call of the Update method.
     bool first;
 
     // The initial (Adam) learning rate.
-    double initialStepSize;
+    ElemType initialStepSize;
 
     // The number of iterations.
     size_t iteration;

include/ensmallen_bits/ada_bound/ams_bound_update.hpp

@@ -96,6 +96,8 @@ class AMSBoundUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -105,11 +107,25 @@ class AMSBoundUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AMSBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0), iteration(0)
+        parent(parent),
+        finalLr(ElemType(parent.finalLr)),
+        gamma(ElemType(parent.gamma)),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        first(true),
+        initialStepSize(0),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
       vImproved.zeros(rows, cols);
+
+      // Check for underflows in conversions.
+      if (gamma == ElemType(0) && parent.gamma != 0.0)
+        gamma = 10 * std::numeric_limits<ElemType>::epsilon();
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -123,40 +139,36 @@ class AMSBoundUpdate
                 const double stepSize,
                 const GradType& gradient)
     {
-      // Convenience typedefs.
-      typedef typename MatType::elem_type ElemType;
-
       // Save the initial step size.
       if (first)
       {
         first = false;
-        initialStepSize = stepSize;
+        initialStepSize = ElemType(stepSize);
       }
 
       // Increment the iteration counter variable.
       ++iteration;
 
       // Decay the first and second moment running average coefficient.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
-      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma * iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma * iteration));
+      const ElemType fl = finalLr * ElemType(stepSize) / initialStepSize;
+      const ElemType lower = fl * (1 - 1 / (gamma * iteration + 1));
+      const ElemType upper = fl * (1 + 1 / (gamma * iteration));
 
       // Element wise maximum of past and present squared gradients.
-      vImproved = arma::max(vImproved, v);
+      vImproved = max(vImproved, v);
 
       // Applies bounds on actual learning rate.
-      iterate -= arma::clamp((stepSize *
-          std::sqrt(biasCorrection2) / biasCorrection1) /
-          (arma::sqrt(vImproved) + parent.epsilon),  lower, upper) % m;
+      iterate -= clamp((ElemType(stepSize) * std::sqrt(biasCorrection2) /
+          biasCorrection1) / (sqrt(vImproved) + epsilon), lower, upper) % m;
     }
 
    private:
@@ -169,11 +181,18 @@ class AMSBoundUpdate
     // The exponential moving average of squared gradient values.
     GradType v;
 
+    // Parameters of the parent, casted to the element type of the problem.
+    ElemType finalLr;
+    ElemType gamma;
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // Whether this is the first call of the Update method.
     bool first;
 
     // The initial (Adam) learning rate.
-    double initialStepSize;
+    ElemType initialStepSize;
 
     // The optimal squared gradient value.
     GradType vImproved;

include/ensmallen_bits/ada_delta/ada_delta.hpp

@@ -98,7 +98,7 @@ class AdaDelta
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ada_delta/ada_delta_update.hpp

@@ -71,6 +71,8 @@ class AdaDeltaUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD optimizer method before the start
      * of the iteration update process. In AdaDelta update policy, the mean
@@ -82,10 +84,16 @@ class AdaDeltaUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdaDeltaUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        rho(ElemType(parent.rho)),
+        epsilon(ElemType(parent.epsilon))
     {
       meanSquaredGradient.zeros(rows, cols);
       meanSquaredGradientDx.zeros(rows, cols);
+
+      // Check for underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -102,17 +110,17 @@ class AdaDeltaUpdate
                 const GradType& gradient)
     {
       // Accumulate gradient.
-      meanSquaredGradient *= parent.rho;
-      meanSquaredGradient += (1 - parent.rho) * (gradient % gradient);
-      GradType dx = arma::sqrt((meanSquaredGradientDx + parent.epsilon) /
-          (meanSquaredGradient + parent.epsilon)) % gradient;
+      meanSquaredGradient *= rho;
+      meanSquaredGradient += (1 - rho) * (gradient % gradient);
+      GradType dx = sqrt((meanSquaredGradientDx + epsilon) /
+          (meanSquaredGradient + epsilon)) % gradient;
 
       // Accumulate updates.
-      meanSquaredGradientDx *= parent.rho;
-      meanSquaredGradientDx += (1 - parent.rho) * (dx % dx);
+      meanSquaredGradientDx *= rho;
+      meanSquaredGradientDx += (1 - rho) * (dx % dx);
 
       // Apply update.
-      iterate -= (stepSize * dx);
+      iterate -= (ElemType(stepSize) * dx);
     }
 
    private:
@@ -124,6 +132,10 @@ class AdaDeltaUpdate
 
     // The delta mean squared gradient matrix.
     GradType meanSquaredGradientDx;
+
+    // Parameters of the update, converted to the matrix element type.
+    ElemType rho;
+    ElemType epsilon;
   };
 
  private:

include/ensmallen_bits/ada_grad/ada_grad.hpp

@@ -94,7 +94,7 @@ class AdaGrad
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ada_grad/ada_grad_update.hpp

@@ -64,6 +64,8 @@ class AdaGradUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD optimizer before the start of the
      * iteration update process. In AdaGrad update policy, squared gradient
@@ -76,10 +78,14 @@ class AdaGradUpdate
      */
     Policy(AdaGradUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
-        squaredGradient(rows, cols)
+        squaredGradient(rows, cols),
+        epsilon(ElemType(parent.epsilon))
     {
       // Initialize an empty matrix for sum of squares of parameter gradient.
       squaredGradient.zeros();
+      // Detect underflow for epsilon and try to address it.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -96,8 +102,8 @@ class AdaGradUpdate
                 const GradType& gradient)
     {
       squaredGradient += (gradient % gradient);
-      iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) +
-          parent.epsilon);
+      iterate -= (ElemType(stepSize) * gradient) / (sqrt(squaredGradient) +
+          epsilon);
     }
 
    private:
@@ -105,6 +111,8 @@ class AdaGradUpdate
     AdaGradUpdate& parent;
     // The squared gradient matrix.
     GradType squaredGradient;
+    // The epsilon value, converted to the element type of the matrix.
+    ElemType epsilon;
   };
 
  private:

include/ensmallen_bits/ada_sqrt/ada_sqrt.hpp

@@ -89,7 +89,7 @@ class AdaSqrt
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ada_sqrt/ada_sqrt_update.hpp

@@ -59,6 +59,8 @@ class AdaSqrtUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD optimizer before the start of the
      * iteration update process. In AdaSqrt update policy, squared gradient
@@ -72,10 +74,14 @@ class AdaSqrtUpdate
     Policy(AdaSqrtUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
         squaredGradient(rows, cols),
+        epsilon(ElemType(parent.epsilon)),
         iteration(0)
     {
       // Initialize an empty matrix for sum of squares of parameter gradient.
       squaredGradient.zeros();
+      // Check for underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -93,10 +99,10 @@ class AdaSqrtUpdate
     {
       ++iteration;
 
-      squaredGradient += arma::square(gradient);
+      squaredGradient += square(gradient);
 
-      iterate -= stepSize * std::sqrt(iteration) * gradient /
-          (squaredGradient + parent.epsilon);
+      iterate -= ElemType(stepSize) * std::sqrt(ElemType(iteration)) *
+          gradient / (squaredGradient + epsilon);
     }
 
    private:
@@ -104,6 +110,8 @@ class AdaSqrtUpdate
     AdaSqrtUpdate& parent;
     // The squared gradient matrix.
     GradType squaredGradient;
+    // Epsilon converted to the element type of the optimization.
+    ElemType epsilon;
     // The number of iterations.
     size_t iteration;
   };

include/ensmallen_bits/adam/adam.hpp

@@ -120,7 +120,7 @@ class AdamType
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/adam/adam_update.hpp

@@ -82,6 +82,8 @@ class AdamUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -92,10 +94,17 @@ class AdamUpdate
      */
     Policy(AdamUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
         iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -113,22 +122,23 @@ class AdamUpdate
       ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * square(gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
       /**
        * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
        * following expression is an approximation of the following actual term;
        * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
        */
-      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-          m / (arma::sqrt(v) + parent.epsilon);
+      iterate -= (ElemType(stepSize) *
+          std::sqrt(biasCorrection2) / biasCorrection1) *
+          m / (sqrt(v) + epsilon);
     }
 
    private:
@@ -141,6 +151,11 @@ class AdamUpdate
     // The exponential moving average of squared gradient values.
     GradType v;
 
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // The number of iterations.
     size_t iteration;
   };

include/ensmallen_bits/adam/adamax_update.hpp

@@ -30,11 +30,11 @@ namespace ens {
  *
  * @code
  * @article{Kingma2014,
- *   author    = {Diederik P. Kingma and Jimmy Ba},
- *   title     = {Adam: {A} Method for Stochastic Optimization},
- *   journal   = {CoRR},
- *   year      = {2014},
- *   url       = {http://arxiv.org/abs/1412.6980}
+ *   author  = {Diederik P. Kingma and Jimmy Ba},
+ *   title   = {Adam: {A} Method for Stochastic Optimization},
+ *   journal = {CoRR},
+ *   year    = {2014},
+ *   url     = {http://arxiv.org/abs/1412.6980}
  * }
  * @endcode
  */
@@ -84,6 +84,8 @@ class AdaMaxUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -94,10 +96,16 @@ class AdaMaxUpdate
      */
     Policy(AdaMaxUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
         iteration(0)
     {
       m.zeros(rows, cols);
       u.zeros(rows, cols);
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -115,17 +123,17 @@ class AdaMaxUpdate
       ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
       // Update the exponentially weighted infinity norm.
-      u *= parent.beta2;
-      u = arma::max(u, arma::abs(gradient));
+      u *= beta2;
+      u = max(u, abs(gradient));
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
 
       if (biasCorrection1 != 0)
-        iterate -= (stepSize / biasCorrection1 * m / (u + parent.epsilon));
+        iterate -= (ElemType(stepSize) / biasCorrection1 * m / (u + epsilon));
     }
 
    private:
@@ -135,6 +143,10 @@ class AdaMaxUpdate
     GradType m;
     // The exponentially weighted infinity norm.
     GradType u;
+    // Tuning parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
     // The number of iterations.
     size_t iteration;
   };

include/ensmallen_bits/adam/amsgrad_update.hpp

@@ -2,7 +2,7 @@
  * @file amsgrad_update.hpp
  * @author Haritha Nair
  *
- * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average
  * optimizer that dynamically adapts over time with guaranteed convergence.
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -25,9 +25,9 @@ namespace ens {
  *
  * @code
  * @article{
- *   title   = {On the convergence of Adam and beyond},
- *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
- *   year    = {2018}
+ *   title = {On the convergence of Adam and beyond},
+ *   url   = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ *   year  = {2018}
  * }
  * @endcode
  */
@@ -77,6 +77,8 @@ class AMSGradUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -87,11 +89,18 @@ class AMSGradUpdate
      */
     Policy(AMSGradUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
         iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
       vImproved.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -109,20 +118,21 @@ class AMSGradUpdate
       ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
       // Element wise maximum of past and present squared gradients.
-      vImproved = arma::max(vImproved, v);
+      vImproved = max(vImproved, v);
 
-      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-                  m / (arma::sqrt(vImproved) + parent.epsilon);
+      iterate -= (ElemType(stepSize) *
+          std::sqrt(biasCorrection2) / biasCorrection1) *
+          m / (sqrt(vImproved) + epsilon);
     }
 
    private:
@@ -138,6 +148,11 @@ class AMSGradUpdate
     // The optimal squared gradient value.
     GradType vImproved;
 
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // The number of iterations.
     size_t iteration;
   };

include/ensmallen_bits/adam/nadam_update.hpp

@@ -85,6 +85,8 @@ class NadamUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the optimizer before the start of the
      * iteration update process.
@@ -96,10 +98,17 @@ class NadamUpdate
     Policy(NadamUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
         cumBeta1(1),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
         iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -117,30 +126,31 @@ class NadamUpdate
       ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * gradient % gradient;
+      v *= beta2;
+      v += (1 - beta2) * gradient % gradient;
 
-      double beta1T = parent.beta1 * (1 - (0.5 *
+      ElemType beta1T = beta1 * (1 - ElemType(0.5 *
           std::pow(0.96, iteration * parent.scheduleDecay)));
 
-      double beta1T1 = parent.beta1 * (1 - (0.5 *
+      ElemType beta1T1 = beta1 * (1 - ElemType(0.5 *
           std::pow(0.96, (iteration + 1) * parent.scheduleDecay)));
 
       cumBeta1 *= beta1T;
 
-      const double biasCorrection1 = 1.0 - cumBeta1;
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
-      const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
+      const ElemType biasCorrection1 = 1 - cumBeta1;
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
+      const ElemType biasCorrection3 = 1 - (cumBeta1 * beta1T1);
 
       /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
        * as arma::sqrt(v) + epsilon
        */
-      iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
-          + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2))
-          / (arma::sqrt(v) + parent.epsilon);
+      iterate -= (ElemType(stepSize) *
+          (((1 - beta1T) / biasCorrection1) * gradient +
+          (beta1T1 / biasCorrection3) * m) * std::sqrt(biasCorrection2)) /
+          (sqrt(v) + epsilon);
     }
 
    private:
@@ -154,7 +164,12 @@ class NadamUpdate
     GradType v;
 
     // The cumulative product of decay coefficients.
-    double cumBeta1;
+    ElemType cumBeta1;
+
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
 
     // The number of iterations.
     size_t iteration;

include/ensmallen_bits/adam/nadamax_update.hpp

@@ -85,6 +85,8 @@ class NadaMaxUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor method is called by the optimizer before the start of
      * the iteration update process.
@@ -96,10 +98,17 @@ class NadaMaxUpdate
     Policy(NadaMaxUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
         cumBeta1(1),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
         iteration(0)
     {
       m.zeros(rows, cols);
       u.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -117,27 +126,27 @@ class NadaMaxUpdate
       ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      u = arma::max(u * parent.beta2, arma::abs(gradient));
+      u = max(u * beta2, abs(gradient));
 
-      double beta1T = parent.beta1 * (1 - (0.5 *
+      ElemType beta1T = beta1 * (1 - ElemType(0.5 *
           std::pow(0.96, iteration * parent.scheduleDecay)));
 
-      double beta1T1 = parent.beta1 * (1 - (0.5 *
+      ElemType beta1T1 = beta1 * (1 - ElemType(0.5 *
           std::pow(0.96, (iteration + 1) * parent.scheduleDecay)));
 
       cumBeta1 *= beta1T;
 
-      const double biasCorrection1 = 1.0 - cumBeta1;
-
-      const double biasCorrection2 = 1.0 - (cumBeta1 * beta1T1);
+      const ElemType biasCorrection1 = 1 - cumBeta1;
+      const ElemType biasCorrection2 = 1 - (cumBeta1 * beta1T1);
 
       if ((biasCorrection1 != 0) && (biasCorrection2 != 0))
       {
-         iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
-             + (beta1T1 / biasCorrection2) * m)) / (u + parent.epsilon);
+        iterate -= (ElemType(stepSize) *
+            (((1 - beta1T) / biasCorrection1) * gradient +
+            (beta1T1 / biasCorrection2) * m)) / (u + epsilon);
       }
     }
 
@@ -152,7 +161,12 @@ class NadaMaxUpdate
     GradType u;
 
     // The cumulative product of decay coefficients.
-    double cumBeta1;
+    ElemType cumBeta1;
+
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
 
     // The number of iterations.
     size_t iteration;

include/ensmallen_bits/adam/optimisticadam_update.hpp

@@ -27,11 +27,11 @@ namespace ens {
  *
  * @code
  * @article{
- *   author  = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis,
- *              Haoyang Zeng},
- *   title   = {Training GANs with Optimism},
- *   year    = {2017},
- *   url     = {https://arxiv.org/abs/1711.00141}
+ *   author = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis,
+ *             Haoyang Zeng},
+ *   title  = {Training GANs with Optimism},
+ *   year   = {2017},
+ *   url    = {https://arxiv.org/abs/1711.00141}
  * }
  * @endcode
  */
@@ -81,6 +81,8 @@ class OptimisticAdamUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -91,11 +93,18 @@ class OptimisticAdamUpdate
      */
     Policy(OptimisticAdamUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
         iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
       g.zeros(rows, cols);
+
+      // Attempt to detect underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -113,18 +122,18 @@ class OptimisticAdamUpdate
       ++iteration;
 
       // And update the iterate.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * arma::square(gradient);
+      v *= beta2;
+      v += (1 - beta2) * square(gradient);
 
-      GradType mCorrected = m / (1.0 - std::pow(parent.beta1, iteration));
-      GradType vCorrected = v / (1.0 - std::pow(parent.beta2, iteration));
+      GradType mCorrected = m / (1 - std::pow(beta1, ElemType(iteration)));
+      GradType vCorrected = v / (1 - std::pow(beta2, ElemType(iteration)));
 
-      GradType update = mCorrected / (arma::sqrt(vCorrected) + parent.epsilon);
+      GradType update = mCorrected / (sqrt(vCorrected) + epsilon);
 
-      iterate -= (2 * stepSize * update - stepSize * g);
+      iterate -= (2 * ElemType(stepSize) * update - ElemType(stepSize) * g);
 
       g = std::move(update);
     }
@@ -142,6 +151,11 @@ class OptimisticAdamUpdate
     // The previous update.
     GradType g;
 
+    // Parameters converted to the element type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // The number of iterations.
     size_t iteration;
   };

include/ensmallen_bits/agemoea/agemoea.hpp

@@ -126,6 +126,33 @@ class AGEMOEA
       MatType& iterate,
       CallbackTypes&&... callbacks);
 
+  /**
+   * Optimize a set of objectives. The initial population is generated using the
+   * starting point. The output is the best generated front.
+   *
+   * @tparam ArbitraryFunctionType std::tuple of multiple objectives.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CubeType The type of cube used to store the front and Pareto set.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param objectives Vector of objective functions to optimize for.
+   * @param iterate Starting point.
+   * @param front The generated front.
+   * @param paretoSet The generated Pareto set.
+   * @param callbacks Callback functions.
+   * @return MatType::elem_type The minimum of the accumulated sum over the
+   *     objective values in the best front.
+   */
+  template<typename MatType,
+           typename CubeType,
+           typename... ArbitraryFunctionType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(
+      std::tuple<ArbitraryFunctionType...>& objectives,
+      MatType& iterate,
+      CubeType& front,
+      CubeType& paretoSet,
+      CallbackTypes&&... callbacks);
+
   //! Get the population size.
   size_t PopulationSize() const { return populationSize; }
   //! Modify the population size.
@@ -166,34 +193,6 @@ class AGEMOEA
   //! Modify value of upperBound.
   arma::vec& UpperBound() { return upperBound; }
 
-  //! Retrieve the Pareto optimal points in variable space. This returns an empty cube
-  //! until `Optimize()` has been called.
-  const arma::cube& ParetoSet() const { return paretoSet; }
-
-  //! Retrieve the best front (the Pareto frontier). This returns an empty cube until
-  //! `Optimize()` has been called.
-  const arma::cube& ParetoFront() const { return paretoFront; }
-
-  /**
-   * Retrieve the best front (the Pareto frontier).  This returns an empty
-   * vector until `Optimize()` has been called.  Note that this function is
-   * deprecated and will be removed in ensmallen 3.x!  Use `ParetoFront()`
-   * instead.
-   */
-  const std::vector<arma::mat>& Front()
-  {
-    if (rcFront.size() == 0)
-    {
-      // Match the old return format.
-      for (size_t i = 0; i < paretoFront.n_slices; ++i)
-      {
-        rcFront.push_back(arma::mat(paretoFront.slice(i)));
-      }
-    }
-
-    return rcFront;
-  }
-
  private:
   /**
    * Evaluate objectives for the elite population.
@@ -205,21 +204,22 @@ class AGEMOEA
    * @param calculatedObjectives Vector to store calculated objectives.
    */
   template<std::size_t I = 0,
-           typename MatType,
+           typename InputMatType,
+           typename ObjectiveMatType,
            typename ...ArbitraryFunctionType>
   typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
-  EvaluateObjectives(std::vector<MatType>&,
+  EvaluateObjectives(std::vector<InputMatType>&,
                      std::tuple<ArbitraryFunctionType...>&,
-                     std::vector<arma::Col<typename MatType::elem_type> >&);
+                     std::vector<ObjectiveMatType>&);
 
   template<std::size_t I = 0,
-           typename MatType,
+           typename InputMatType,
+           typename ObjectiveMatType,
            typename ...ArbitraryFunctionType>
   typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
-  EvaluateObjectives(std::vector<MatType>& population,
+  EvaluateObjectives(std::vector<InputMatType>& population,
                      std::tuple<ArbitraryFunctionType...>& objectives,
-                     std::vector<arma::Col<typename MatType::elem_type> >&
-                     calculatedObjectives);
+                     std::vector<ObjectiveMatType>& calculatedObjectives);
 
   /**
    * Reproduce candidates from the elite population to generate a new
@@ -283,7 +283,8 @@ class AGEMOEA
   void FastNonDominatedSort(
       std::vector<std::vector<size_t> >& fronts,
       std::vector<size_t>& ranks,
-      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives);
+      std::vector<arma::Col<typename MatType::elem_type> >&
+          calculatedObjectives);
 
   /**
    * Operator to check if one candidate Pareto-dominates the other.
@@ -304,17 +305,18 @@ class AGEMOEA
       size_t candidateP,
       size_t candidateQ);
 
- /**
-  * Assigns Survival Score metric for sorting.
-  *
-  * @param front The previously generated Pareto fronts.
-  * @param idealPoint The ideal point of teh first front.
-  * @param calculatedObjectives The previously calculated objectives.
-  * @param survivalScore The Survival Score vector to be updated for each individual in the population.
-  * @param normalize The normlization vector of the fronts.
-  * @param dimension The dimension of the first front.
-  * @param fNum teh current front index.
-  */
+  /**
+   * Assigns Survival Score metric for sorting.
+   *
+   * @param front The previously generated Pareto fronts.
+   * @param idealPoint The ideal point of teh first front.
+   * @param calculatedObjectives The previously calculated objectives.
+   * @param survivalScore The Survival Score vector to be updated for each
+   *     individual in the population.
+   * @param normalize The normlization vector of the fronts.
+   * @param dimension The dimension of the first front.
+   * @param fNum teh current front index.
+   */
   template <typename MatType>
   void SurvivalScoreAssignment(
       const std::vector<size_t>& front,
@@ -322,7 +324,7 @@ class AGEMOEA
       std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
       std::vector<typename MatType::elem_type>& survivalScore,
       arma::Col<typename MatType::elem_type>& normalize,
-      double& dimension,
+      typename MatType::elem_type& dimension,
       size_t fNum);
 
   /**
@@ -338,7 +340,7 @@ class AGEMOEA
    *     being sorted.
    * @param ranks The previously calculated ranks.
    * @param survivalScore The Survival score for each individual in
-   *	the population.
+   *     the population.
    * @return true if the first candidate is preferred, otherwise, false.
    */
   template<typename MatType>
@@ -347,37 +349,39 @@ class AGEMOEA
       size_t idxQ,
       const std::vector<size_t>& ranks,
       const std::vector<typename MatType::elem_type>& survivalScore);
-  
- /**
-  * Normalizes the front given the extreme points in the current front.
-  *
-  * @tparam The type of population datapoints.
-  * @param calculatedObjectives The current population evaluated objectives.
-  * @param normalization The normalizing vector.
-  * @param front The previously generated Pareto front.
-  * @param extreme The indexes of the extreme points in the front.
-  */
- template <typename MatType>
- void NormalizeFront(
-     std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
-     arma::Col<typename MatType::elem_type>& normalization,
-     const std::vector<size_t>& front,
-     const arma::Row<size_t>& extreme);
- 
- /**
-  * Get the geometry information p of Lp norm (p > 0).
-  *
-  * @param calculatedObjectives The current population evaluated objectives.
-  * @param front The previously generated Pareto fronts.
-  * @param extreme The indexes of the extreme points in the front.
-  * @return The variable p in the Lp norm that best fits the geometry of the current front.
-  */
- template <typename MatType>
- double GetGeometry(
-      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+
+  /**
+   * Normalizes the front given the extreme points in the current front.
+   *
+   * @tparam The type of population datapoints.
+   * @param calculatedObjectives The current population evaluated objectives.
+   * @param normalization The normalizing vector.
+   * @param front The previously generated Pareto front.
+   * @param extreme The indexes of the extreme points in the front.
+   */
+  template <typename MatType>
+  void NormalizeFront(
+      std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
+      arma::Col<typename MatType::elem_type>& normalization,
       const std::vector<size_t>& front,
       const arma::Row<size_t>& extreme);
-  
+
+  /**
+   * Get the geometry information p of Lp norm (p > 0).
+   *
+   * @param calculatedObjectives The current population evaluated objectives.
+   * @param front The previously generated Pareto fronts.
+   * @param extreme The indexes of the extreme points in the front.
+   * @return The variable p in the Lp norm that best fits the geometry of the
+   *     current front.
+   */
+  template <typename MatType>
+  typename MatType::elem_type GetGeometry(
+      std::vector<arma::Col<typename MatType::elem_type> >&
+          calculatedObjectives,
+      const std::vector<size_t>& front,
+      const arma::Row<size_t>& extreme);
+
   /**
    * Finds the pairwise Lp distance between all the points in the front.
    *
@@ -389,13 +393,14 @@ class AGEMOEA
   template <typename MatType>
   void PairwiseDistance(
       MatType& final,
-      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      std::vector<arma::Col<typename MatType::elem_type> >&
+          calculatedObjectives,
       const std::vector<size_t>& front,
-      double dimension);
+      const typename MatType::elem_type dimension);
 
   /**
    * Finding the indexes of the extreme points in the front.
-   *  
+   *
    * @param indexes vector containing the slected indexes.
    * @param calculatedObjectives The current population objectives.
    * @param front The front of the current generation.
@@ -405,32 +410,37 @@ class AGEMOEA
       arma::Row<size_t>& indexes,
       std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
       const std::vector<size_t>& front);
-  
+
   /**
    * Finding the distance of each point in the front from the line formed
    * by pointA and pointB.
-   * 
-   * @param distance The vector containing the distances of the points in the fron from the line.
-   * @param calculatedObjectives Reference to the current population evaluated Objectives.
+   *
+   * @param distance The vector containing the distances of the points in the
+   *    from from the line.
+   * @param calculatedObjectives Reference to the current population evaluated
+   *    objectives.
    * @param front The front of the current generation(indices of population).
    * @param pointA The first point on the line.
    * @param pointB The second point on the line.
-  */
+   */
   template <typename MatType>
   void PointToLineDistance(
       arma::Row<typename MatType::elem_type>& distances,
-      std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
+      std::vector<arma::Col<typename MatType::elem_type> >&
+          calculatedObjectives,
       const std::vector<size_t>& front,
       const arma::Col<typename MatType::elem_type>& pointA,
       const arma::Col<typename MatType::elem_type>& pointB);
-  
+
   /**
-   * Find the Diversity score corresponding the solution S using the selected set.
-   * 
+   * Find the Diversity score corresponding the solution S using the selected
+   * set.
+   *
    * @param selected The current selected set.
    * @param pairwiseDistance The current pairwise distance for the whole front.
    * @param S The relative index of S being considered within the front.
-   * @return The diversity score for S which the sum of the two smallest elements.
+   * @return The diversity score for S which the sum of the two smallest
+   *     elements.
   */
  template <typename MatType>
  typename MatType::elem_type DiversityScore(std::set<size_t>& selected,
@@ -467,19 +477,6 @@ class AGEMOEA
 
   //! Upper bound of the initial swarm.
   arma::vec upperBound;
-
-  //! The set of all the Pareto optimal points.
-  //! Stored after Optimize() is called.
-  arma::cube paretoSet;
-
-  //! The set of all the Pareto optimal objective vectors.
-  //! Stored after Optimize() is called.
-  arma::cube paretoFront;
-
-  //! A different representation of the Pareto front, for reverse compatibility
-  //! purposes.  This can be removed when ensmallen 3.x is released!  (Along
-  //! with `Front()`.)  This is only populated when `Front()` is called.
-  std::vector<arma::mat> rcFront;
 };
 
 } // namespace ens

include/ensmallen_bits/agemoea/agemoea_impl.hpp

@@ -67,6 +67,24 @@ typename MatType::elem_type AGEMOEA::Optimize(
     std::tuple<ArbitraryFunctionType...>& objectives,
     MatType& iterateIn,
     CallbackTypes&&... callbacks)
+{
+  typedef typename ForwardType<MatType>::bcube CubeType;
+  CubeType paretoFront, paretoSet;
+  return Optimize(objectives, iterateIn, paretoFront, paretoSet,
+      std::forward<CallbackTypes>(callbacks)...);
+}
+
+//! Optimize the function.
+template<typename MatType,
+         typename CubeType,
+         typename... ArbitraryFunctionType,
+         typename... CallbackTypes>
+typename MatType::elem_type AGEMOEA::Optimize(
+    std::tuple<ArbitraryFunctionType...>& objectives,
+    MatType& iterateIn,
+    CubeType& paretoFrontIn,
+    CubeType& paretoSetIn,
+    CallbackTypes&&... callbacks)
 {
   // Make sure for evolution to work at least four candidates are present.
   if (populationSize < 4 && populationSize % 4 != 0)
@@ -78,6 +96,8 @@ typename MatType::elem_type AGEMOEA::Optimize(
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
   typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename ForwardType<MatType>::bcol BaseColType;
+  typedef typename ForwardType<CubeType>::bmat CubeBaseMatType;
 
   BaseMatType& iterate = (BaseMatType&) iterateIn;
 
@@ -104,7 +124,7 @@ typename MatType::elem_type AGEMOEA::Optimize(
   numVariables = iterate.n_rows;
 
   // Cache calculated objectives.
-  std::vector<arma::Col<ElemType> > calculatedObjectives(populationSize);
+  std::vector<BaseColType> calculatedObjectives(populationSize);
 
   // Population size reserved to 2 * populationSize + 1 to accommodate
   // for the size of intermediate candidate population.
@@ -120,8 +140,8 @@ typename MatType::elem_type AGEMOEA::Optimize(
   std::vector<size_t> ranks;
 
   //! Useful temporaries for float-like comparisons.
-  const BaseMatType castedLowerBound = arma::conv_to<BaseMatType>::from(lowerBound);
-  const BaseMatType castedUpperBound = arma::conv_to<BaseMatType>::from(upperBound);
+  const BaseMatType castedLowerBound = conv_to<BaseMatType>::from(lowerBound);
+  const BaseMatType castedUpperBound = conv_to<BaseMatType>::from(upperBound);
 
   // Controls early termination of the optimization process.
   bool terminate = false;
@@ -131,10 +151,10 @@ typename MatType::elem_type AGEMOEA::Optimize(
   for (size_t i = 0; i < populationSize; i++)
   {
     population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
-        iterate.n_cols) - 0.5 + iterate);
+        iterate.n_cols) - ElemType(0.5) + iterate);
 
     // Constrain all genes to be within bounds.
-    population[i] = arma::min(arma::max(population[i], castedLowerBound), 
+    population[i] = min(max(population[i], castedLowerBound),
         castedUpperBound);
   }
 
@@ -152,26 +172,24 @@ typename MatType::elem_type AGEMOEA::Optimize(
     // Evaluate the objectives for the new population.
     calculatedObjectives.resize(population.size());
     std::fill(calculatedObjectives.begin(), calculatedObjectives.end(),
-        arma::Col<ElemType>(numObjectives, arma::fill::zeros));
+        BaseColType(numObjectives, GetFillType<MatType>::zeros));
     EvaluateObjectives(population, objectives, calculatedObjectives);
 
     // Perform fast non dominated sort on P_t ∪ G_t.
     ranks.resize(population.size());
     FastNonDominatedSort<BaseMatType>(fronts, ranks, calculatedObjectives);
-    
+
     arma::Col<ElemType> idealPoint(calculatedObjectives[fronts[0][0]]);
     for (size_t index = 1; index < fronts[0].size(); index++)
     {
-      idealPoint = arma::min(idealPoint, 
-          calculatedObjectives[fronts[0][index]]);
+      idealPoint = min(idealPoint, calculatedObjectives[fronts[0][index]]);
     }
 
     // Perform survival score assignment.
     survivalScore.resize(population.size());
     std::fill(survivalScore.begin(), survivalScore.end(), 0.);
-    double dimension;
-    arma::Col<typename MatType::elem_type> normalize(numObjectives, 
-        arma::fill::zeros);
+    ElemType dimension;
+    BaseColType normalize(numObjectives, GetFillType<MatType>::zeros);
     for (size_t fNum = 0; fNum < fronts.size(); fNum++)
     {
       SurvivalScoreAssignment<BaseMatType>(fronts[fNum], idealPoint,
@@ -186,16 +204,16 @@ typename MatType::elem_type AGEMOEA::Optimize(
             size_t idxP{}, idxQ{};
             for (size_t i = 0; i < population.size(); i++)
             {
-              if (arma::approx_equal(population[i], candidateP, 
-                  "absdiff", epsilon))
+              if (approx_equal(population[i], candidateP, "absdiff",
+                  ElemType(epsilon)))
                 idxP = i;
 
-              if (arma::approx_equal(population[i], candidateQ, 
-                  "absdiff", epsilon))
+              if (approx_equal(population[i], candidateQ, "absdiff",
+                  ElemType(epsilon)))
                 idxQ = i;
             }
 
-            return SurvivalScoreOperator<BaseMatType>(idxP, idxQ, ranks, 
+            return SurvivalScoreOperator<BaseMatType>(idxP, idxQ, ranks,
                 survivalScore);
           }
     );
@@ -209,29 +227,24 @@ typename MatType::elem_type AGEMOEA::Optimize(
   }
   EvaluateObjectives(population, objectives, calculatedObjectives);
   // Set the candidates from the Pareto Set as the output.
-  paretoSet.set_size(population[0].n_rows, population[0].n_cols, 
+  paretoSetIn.set_size(population[0].n_rows, population[0].n_cols,
       population.size());
   // The Pareto Set is stored, can be obtained via ParetoSet() getter.
   for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
   {
-    paretoSet.slice(solutionIdx) =
-      arma::conv_to<arma::mat>::from(population[solutionIdx]);
+    paretoSetIn.slice(solutionIdx) =
+      conv_to<CubeBaseMatType>::from(population[solutionIdx]);
   }
 
   // Set the candidates from the Pareto Front as the output.
-  paretoFront.set_size(calculatedObjectives[0].n_rows, 
+  paretoFrontIn.set_size(calculatedObjectives[0].n_rows,
       calculatedObjectives[0].n_cols, population.size());
-  // The Pareto Front is stored, can be obtained via ParetoFront() getter.
   for (size_t solutionIdx = 0; solutionIdx < population.size(); ++solutionIdx)
   {
-    paretoFront.slice(solutionIdx) =
-      arma::conv_to<arma::mat>::from(calculatedObjectives[solutionIdx]);
+    paretoFrontIn.slice(solutionIdx) =
+      conv_to<CubeBaseMatType>::from(calculatedObjectives[solutionIdx]);
   }
 
-  // Clear rcFront, in case it is later requested by the user for reverse
-  // compatibility reasons.
-  rcFront.clear();
-
   // Assign iterate to first element of the Pareto Set.
   iterate = population[fronts[0][0]];
 
@@ -239,57 +252,62 @@ typename MatType::elem_type AGEMOEA::Optimize(
 
   ElemType performance = std::numeric_limits<ElemType>::max();
 
-  for (const arma::Col<ElemType>& objective: calculatedObjectives)
-    if (arma::accu(objective) < performance)
-      performance = arma::accu(objective);
+  for (const BaseColType& objective: calculatedObjectives)
+    if (accu(objective) < performance)
+      performance = accu(objective);
 
   return performance;
 }
 
 //! No objectives to evaluate.
 template<std::size_t I,
-         typename MatType,
+         typename InputMatType,
+         typename ObjectiveMatType,
          typename ...ArbitraryFunctionType>
 typename std::enable_if<I == sizeof...(ArbitraryFunctionType), void>::type
 AGEMOEA::EvaluateObjectives(
-    std::vector<MatType>&,
+    std::vector<InputMatType>&,
     std::tuple<ArbitraryFunctionType...>&,
-    std::vector<arma::Col<typename MatType::elem_type> >&)
+    std::vector<ObjectiveMatType>&)
 {
   // Nothing to do here.
 }
 
 //! Evaluate the objectives for the entire population.
 template<std::size_t I,
-         typename MatType,
+         typename InputMatType,
+         typename ObjectiveMatType,
          typename ...ArbitraryFunctionType>
 typename std::enable_if<I < sizeof...(ArbitraryFunctionType), void>::type
 AGEMOEA::EvaluateObjectives(
-    std::vector<MatType>& population,
+    std::vector<InputMatType>& population,
     std::tuple<ArbitraryFunctionType...>& objectives,
-    std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives)
+    std::vector<ObjectiveMatType>& calculatedObjectives)
 {
   for (size_t i = 0; i < population.size(); i++)
   {
     calculatedObjectives[i](I) = std::get<I>(objectives).Evaluate(population[i]);
-    EvaluateObjectives<I+1, MatType, ArbitraryFunctionType...>(population, objectives,
+    EvaluateObjectives<I+1, InputMatType, ObjectiveMatType,
+        ArbitraryFunctionType...>(population, objectives,
         calculatedObjectives);
   }
 }
 
 //! Reproduce and generate new candidates.
-template<typename MatType>
-inline void AGEMOEA::BinaryTournamentSelection(std::vector<MatType>& population,
-                                             const MatType& lowerBound,
-                                             const MatType& upperBound)
+template<typename InputMatType>
+inline void AGEMOEA::BinaryTournamentSelection(std::vector<InputMatType>& population,
+                                               const InputMatType& lowerBound,
+                                               const InputMatType& upperBound)
 {
-  std::vector<MatType> children;
+  std::vector<InputMatType> children;
 
   while (children.size() < population.size())
   {
     // Choose two random parents for reproduction from the elite population.
-    size_t indexA = arma::randi<size_t>(arma::distr_param(0, populationSize - 1));
-    size_t indexB = arma::randi<size_t>(arma::distr_param(0, populationSize - 1));
+    size_t indexA = arma::randi<size_t>(
+        arma::distr_param(0, populationSize - 1));
+    size_t indexB = arma::randi<size_t>(
+        arma::distr_param(0, populationSize - 1));
 
     // Make sure that the parents differ.
     if (indexA == indexB)
@@ -301,10 +319,10 @@ inline void AGEMOEA::BinaryTournamentSelection(std::vector<MatType>& population,
     }
 
     // Initialize the children to the respective parents.
-    MatType childA = population[indexA], childB = population[indexB];
+    InputMatType childA = population[indexA], childB = population[indexB];
 
     if (arma::randu() <= crossoverProb)
-      Crossover(childA, childB, population[indexA], population[indexB], 
+      Crossover(childA, childB, population[indexA], population[indexB],
                 lowerBound, upperBound);
 
     Mutate(childA, 1.0 / static_cast<double>(numVariables),
@@ -318,68 +336,74 @@ inline void AGEMOEA::BinaryTournamentSelection(std::vector<MatType>& population,
   }
 
   // Add the candidates to the elite population.
-  population.insert(std::end(population), std::begin(children), std::end(children));
+  population.insert(std::end(population), std::begin(children),
+      std::end(children));
 }
 
 //! Perform simulated binary crossover (SBX) of genes for the children.
-template<typename MatType>
-inline void AGEMOEA::Crossover(MatType& childA,
-                               MatType& childB,
-                               const MatType& parentA,
-                               const MatType& parentB,
-                               const MatType& lowerBound,
-                               const MatType& upperBound)
+template<typename InputMatType>
+inline void AGEMOEA::Crossover(InputMatType& childA,
+                               InputMatType& childB,
+                               const InputMatType& parentA,
+                               const InputMatType& parentB,
+                               const InputMatType& lowerBound,
+                               const InputMatType& upperBound)
 {
-    //! Generates a child from two parent individuals
-    // according to the polynomial probability distribution.
-    arma::Cube<typename MatType::elem_type> parents(parentA.n_rows, 
-      parentA.n_cols, 2);
-    parents.slice(0) = parentA;
-    parents.slice(1) = parentB;
-    MatType current_min =  arma::min(parents, 2);
-    MatType current_max =  arma::max(parents, 2);
+  typedef typename InputMatType::elem_type ElemType;
+  typedef typename ForwardType<InputMatType>::bcube BaseCubeType;
+  typedef typename ForwardType<InputMatType>::umat UMatType;
 
-    if (arma::accu(parentA - parentB < 1e-14))
-    {
-      childA = parentA;
-      childB = parentB;
-      return;
-    }
-    MatType current_diff = current_max - current_min;
-    current_diff.transform( [](typename MatType::elem_type val)
-      { return (val < 1e-10 ? 1e-10:val); } );
+  // Generates a child from two parent individuals
+  // according to the polynomial probability distribution.
+  BaseCubeType parents(parentA.n_rows,
+    parentA.n_cols, 2);
+  parents.slice(0) = parentA;
+  parents.slice(1) = parentB;
+  InputMatType current_min = min(parents, 2);
+  InputMatType current_max = max(parents, 2);
 
-    // Calculating beta used for the final crossover.
-    MatType beta1 = 1 + 2.0 * (current_min - lowerBound) / current_diff;
-    MatType beta2 = 1 + 2.0 * (upperBound - current_max) / current_diff;
-    MatType alpha1 = 2 - arma::pow(beta1, -(eta + 1));
-    MatType alpha2 = 2 - arma::pow(beta2, -(eta + 1));
+  if (accu(parentA - parentB < ElemType(1e-14)))
+  {
+    childA = parentA;
+    childB = parentB;
+    return;
+  }
+  InputMatType current_diff = current_max - current_min;
+  current_diff.transform( [](ElemType val)
+      { return (val < ElemType(1e-10) ? ElemType(1e-10) : val); } );
 
-    MatType us(arma::size(alpha1), arma::fill::randu);
-    arma::umat mask1 = us > (1.0 / alpha1); 
-    MatType betaq1 = arma::pow(us % alpha1, 1. / (eta + 1));
-    betaq1 = betaq1 % (mask1 != 1.0) + arma::pow((1.0 / (2.0 - us % alpha1)), 
-        1.0 / (eta + 1)) % mask1;
-    arma::umat mask2 = us > (1.0 / alpha2);
-    MatType betaq2 = arma::pow(us % alpha2, 1 / (eta + 1));
-    betaq2 = betaq2 % (mask1 != 1.0) + arma::pow((1.0 / (2.0 - us % alpha2)), 
-        1.0 / (eta + 1)) % mask2;
+  // Calculating beta used for the final crossover.
+  InputMatType beta1 = 1 + 2 * (current_min - lowerBound) / current_diff;
+  InputMatType beta2 = 1 + 2 * (upperBound - current_max) / current_diff;
+  InputMatType alpha1 = 2 - pow(beta1, -(eta + 1));
+  InputMatType alpha2 = 2 - pow(beta2, -(eta + 1));
 
-    // Variables after the cross over for all of them.
-    MatType c1 = 0.5 * ((current_min + current_max) - betaq1 % current_diff);
-    MatType c2 = 0.5 * ((current_min + current_max) + betaq2 % current_diff);
-    c1 = arma::min(arma::max(c1, lowerBound), upperBound);
-    c2 = arma::min(arma::max(c2, lowerBound), upperBound);
-    
-    // Decision for the crossover between the two parents for each variable.
-    us.randu();
-    childA = parentA % (us <= 0.5);
-    childB = parentB % (us <= 0.5);
-    us.randu();
-    childA = childA + c1 % ((us <= 0.5) % (childA == 0));
-    childA = childA + c2 % ((us > 0.5) % (childA == 0));
-    childB = childB + c2 % ((us <= 0.5) % (childB == 0));
-    childB = childB + c1 % ((us > 0.5) % (childB == 0));
+  InputMatType us(size(alpha1), GetFillType<InputMatType>::randu);
+
+  UMatType mask1 = us > (1 / alpha1);
+  InputMatType betaq1 = pow(us % alpha1, 1. / (eta + 1));
+  betaq1 = betaq1 % (mask1 != 1) + pow((1 / (2 - us % alpha1)),
+      1 / (eta + 1)) % mask1;
+  UMatType mask2 = us > (1 / alpha2);
+  InputMatType betaq2 = pow(us % alpha2, 1 / (eta + 1));
+  betaq2 = betaq2 % (mask1 != 1) + pow((1 / (2 - us % alpha2)),
+      1 / (eta + 1)) % mask2;
+
+  // Variables after the cross over for all of them.
+  InputMatType c1 = ((current_min + current_max) - betaq1 % current_diff) / 2;
+  InputMatType c2 = ((current_min + current_max) + betaq2 % current_diff) / 2;
+  c1 = min(max(c1, lowerBound), upperBound);
+  c2 = min(max(c2, lowerBound), upperBound);
+
+  // Decision for the crossover between the two parents for each variable.
+  us.randu();
+  childA = parentA % (us <= ElemType(0.5));
+  childB = parentB % (us <= ElemType(0.5));
+  us.randu();
+  childA = childA + c1 % ((us <= ElemType(0.5)) % (childA == 0));
+  childA = childA + c2 % ((us >  ElemType(0.5)) % (childA == 0));
+  childB = childB + c2 % ((us <= ElemType(0.5)) % (childB == 0));
+  childB = childB + c1 % ((us >  ElemType(0.5)) % (childB == 0));
 }
 
 //! Perform Polynomial mutation of the candidate.
@@ -389,39 +413,40 @@ inline void AGEMOEA::Mutate(MatType& candidate,
                             const MatType& lowerBound,
                             const MatType& upperBound)
 {
-    const size_t numVariables = candidate.n_rows;
-    for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+  const size_t numVariables = candidate.n_rows;
+  for (size_t geneIdx = 0; geneIdx < numVariables; ++geneIdx)
+  {
+    // Should this gene be mutated?
+    if (arma::randu() > mutationRate)
+      continue;
+
+    const double geneRange = upperBound(geneIdx) - lowerBound(geneIdx);
+    // Normalised distance from the bounds.
+    const double lowerDelta = (candidate(geneIdx)
+        - lowerBound(geneIdx)) / geneRange;
+    const double upperDelta = (upperBound(geneIdx)
+        - candidate(geneIdx)) / geneRange;
+    const double mutationPower = 1. / (distributionIndex + 1.0);
+    const double rand = arma::randu();
+    double value, perturbationFactor;
+    if (rand < 0.5)
     {
-      // Should this gene be mutated?
-      if (arma::randu() > mutationRate)
-        continue;
-
-      const double geneRange = upperBound(geneIdx) - lowerBound(geneIdx);
-      // Normalised distance from the bounds.
-      const double lowerDelta = (candidate(geneIdx) 
-          - lowerBound(geneIdx)) / geneRange;
-      const double upperDelta = (upperBound(geneIdx) 
-          - candidate(geneIdx)) / geneRange;
-      const double mutationPower = 1. / (distributionIndex + 1.0);
-      const double rand = arma::randu();
-      double value, perturbationFactor;
-      if (rand < 0.5)
-      {
-        value = 2.0 * rand + (1.0 - 2.0 * rand) *
-            std::pow(upperDelta, distributionIndex + 1.0);
-        perturbationFactor = std::pow(value, mutationPower) - 1.0;
-      }
-      else
-      {
-        value = 2.0 * (1.0 - rand) + 2.0 *(rand - 0.5) *
-            std::pow(lowerDelta, distributionIndex + 1.0);
-        perturbationFactor = 1.0 - std::pow(value, mutationPower);
-      }
-
-      candidate(geneIdx) += perturbationFactor * geneRange;
+      value = 2.0 * rand + (1.0 - 2.0 * rand) *
+          std::pow(upperDelta, distributionIndex + 1.0);
+      perturbationFactor = std::pow(value, mutationPower) - 1.0;
     }
-    //! Enforce bounds.
-    candidate = arma::min(arma::max(candidate, lowerBound), upperBound);
+    else
+    {
+      value = 2.0 * (1.0 - rand) + 2.0 *(rand - 0.5) *
+          std::pow(lowerDelta, distributionIndex + 1.0);
+      perturbationFactor = 1.0 - std::pow(value, mutationPower);
+    }
+
+    candidate(geneIdx) +=
+        typename MatType::elem_type(perturbationFactor * geneRange);
+  }
+  //! Enforce bounds.
+  candidate = min(max(candidate, lowerBound), upperBound);
 }
 
 template <typename MatType>
@@ -431,9 +456,9 @@ inline void AGEMOEA::NormalizeFront(
       const std::vector<size_t>& front,
       const arma::Row<size_t>& extreme)
 {
-  arma::Mat<typename MatType::elem_type> vectorizedObjectives(numObjectives, 
+  arma::Mat<typename MatType::elem_type> vectorizedObjectives(numObjectives,
       front.size());
-  arma::Mat<typename MatType::elem_type> vectorizedExtremes(numObjectives, 
+  arma::Mat<typename MatType::elem_type> vectorizedExtremes(numObjectives,
       extreme.n_elem);
   for (size_t i = 0; i < front.size(); i++)
   {
@@ -441,7 +466,7 @@ inline void AGEMOEA::NormalizeFront(
   }
   for (size_t i = 0; i < extreme.n_elem; i++)
   {
-    vectorizedExtremes.col(i) = calculatedObjectives[front[extreme[i]]]; 
+    vectorizedExtremes.col(i) = calculatedObjectives[front[extreme[i]]];
   }
 
   if (front.size() < numObjectives)
@@ -474,9 +499,9 @@ inline void AGEMOEA::NormalizeFront(
   }
   else
   {
-    normalization = 1. / hyperplane;   
+    normalization = 1. / hyperplane;
     if (normalization.has_inf() || normalization.has_nan())
-    {    
+    {
       normalization = arma::max(vectorizedObjectives, 1);
     }
   }
@@ -484,26 +509,29 @@ inline void AGEMOEA::NormalizeFront(
 }
 
 template <typename MatType>
-inline double AGEMOEA::GetGeometry(
+inline typename MatType::elem_type AGEMOEA::GetGeometry(
     std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
     const std::vector<size_t>& front,
     const arma::Row<size_t>& extreme)
 {
-  arma::Row<typename MatType::elem_type> d;
-  arma::Col<typename MatType::elem_type> zero(numObjectives, arma::fill::zeros);
-  arma::Col<typename MatType::elem_type> one(numObjectives, arma::fill::ones);
+  typedef typename MatType::elem_type ElemType;
 
-  PointToLineDistance<MatType> (d, calculatedObjectives, front, zero, one);
+  arma::Row<ElemType> d;
+  arma::Col<ElemType> zero(numObjectives, arma::fill::zeros);
+  arma::Col<ElemType> one(numObjectives, arma::fill::ones);
+
+  PointToLineDistance<MatType>(d, calculatedObjectives, front, zero, one);
 
   for (size_t i = 0; i < extreme.size(); i++)
   {
-    d[extreme[i]] = arma::datum::inf;
+    d[extreme[i]] = arma::Datum<ElemType>::inf;
   }
+
   size_t index = arma::index_min(d);
-  double avg = arma::accu(calculatedObjectives[front[index]]) / static_cast<double> (numObjectives); 
-  double p = std::log(numObjectives) / std::log(1.0 / avg);
-  if (p <= 0.1 || std::isnan(p)) 
-    p = 1.0;
+  ElemType avg = accu(calculatedObjectives[front[index]]) / numObjectives;
+  ElemType p = std::log(ElemType(numObjectives)) / std::log(1 / avg);
+  if (p <= ElemType(0.1) || std::isnan(p))
+    p = 1;
 
   return p;
 }
@@ -514,13 +542,15 @@ inline void AGEMOEA::PairwiseDistance(
     MatType& f,
     std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
     const std::vector<size_t>& front,
-    double dimension)
-{ 
+    const typename MatType::elem_type dimension)
+{
   for (size_t i = 0; i < front.size(); i++)
   {
     for (size_t j = i + 1; j < front.size(); j++)
     {
-      f(i, j) = std::pow(arma::accu(arma::pow(arma::abs(calculatedObjectives[front[i]] - calculatedObjectives[front[j]]), dimension)), 1.0 / dimension);
+      f(i, j) = std::pow(accu(pow(abs(
+          calculatedObjectives[front[i]] - calculatedObjectives[front[j]]),
+          dimension)), 1 / dimension);
       f(j, i) = f(i, j);
     }
   }
@@ -529,12 +559,12 @@ inline void AGEMOEA::PairwiseDistance(
 //! Find the index of the of the extreme points in the given front.
 template <typename MatType>
 void AGEMOEA::FindExtremePoints(
-    arma::Row<size_t>& indexes, 
+    arma::Row<size_t>& indexes,
     std::vector<arma::Col<typename MatType::elem_type> >& calculatedObjectives,
     const std::vector<size_t>& front)
 {
   typedef typename MatType::elem_type ElemType;
-  
+
   if (numObjectives >= front.size())
   {
     indexes = arma::linspace<arma::Row<size_t>>(0, front.size() - 1, front.size());
@@ -567,13 +597,13 @@ void AGEMOEA::PointToLineDistance(
 {
   typedef typename MatType::elem_type ElemType;
   arma::Row<ElemType> distancesTemp(front.size());
-  arma::Col<ElemType> ba = pointB - pointA; 
+  arma::Col<ElemType> ba = pointB - pointA;
   arma::Col<ElemType> pa;
 
   for (size_t i = 0; i < front.size(); i++)
   {
     size_t ind = front[i];
- 
+
     pa = (calculatedObjectives[ind] - pointA);
     double t = arma::dot(pa, ba) / arma::dot(ba, ba);
     distancesTemp[i] = arma::accu(arma::pow((pa - t * ba), 2));
@@ -660,7 +690,7 @@ inline bool AGEMOEA::Dominates(
       allBetterOrEqual = false;
 
     // P is better than Q for the i-th objective function.
-    else if (calculatedObjectives[candidateP](i) < 
+    else if (calculatedObjectives[candidateP](i) <
         calculatedObjectives[candidateQ](i))
       atleastOneBetter = true;
   }
@@ -674,7 +704,7 @@ inline typename MatType::elem_type AGEMOEA::DiversityScore(
     std::set<size_t>& selected,
     const MatType& pairwiseDistance,
     size_t S)
-{ 
+{
   typedef typename MatType::elem_type ElemType;
   ElemType m = arma::datum::inf;
   ElemType m1 = arma::datum::inf;
@@ -682,7 +712,7 @@ inline typename MatType::elem_type AGEMOEA::DiversityScore(
   for (it = selected.begin(); it != selected.end(); it++)
   {
     if (*it == S){ continue; }
-    if (pairwiseDistance(S, *it) < m) 
+    if (pairwiseDistance(S, *it) < m)
     {
       m1 = m;
       m = pairwiseDistance(S, *it);
@@ -705,7 +735,7 @@ inline void AGEMOEA::SurvivalScoreAssignment(
     std::vector<arma::Col<typename MatType::elem_type>>& calculatedObjectives,
     std::vector<typename MatType::elem_type>& survivalScore,
     arma::Col<typename MatType::elem_type>& normalize,
-    double& dimension,
+    typename MatType::elem_type& dimension,
     size_t fNum)
 {
   typedef typename MatType::elem_type ElemType;
@@ -718,12 +748,12 @@ inline void AGEMOEA::SurvivalScoreAssignment(
       dimension = 1;
       arma::Row<size_t> extreme(numObjectives, arma::fill::zeros);
       NormalizeFront<MatType>(calculatedObjectives, normalize, front, extreme);
-      return; 
+      return;
     }
 
     for (size_t index = 0; index < front.size(); index++)
     {
-      calculatedObjectives[front[index]] = calculatedObjectives[front[index]] 
+      calculatedObjectives[front[index]] = calculatedObjectives[front[index]]
           - idealPoint;
     }
 
@@ -733,22 +763,21 @@ inline void AGEMOEA::SurvivalScoreAssignment(
 
     for (size_t index = 0; index < front.size(); index++)
     {
-      calculatedObjectives[front[index]] = calculatedObjectives[front[index]] 
+      calculatedObjectives[front[index]] = calculatedObjectives[front[index]]
           / normalize;
     }
 
     std::set<size_t> selected;
     std::set<size_t> remaining;
-    
+
     // Create the selected and remaining sets.
     for (size_t index: extreme)
-    { 
+    {
       selected.insert(index);
-      survivalScore[front[index]] = arma::datum::inf;
+      survivalScore[front[index]] = arma::Datum<ElemType>::inf;
     }
 
-    dimension = GetGeometry<MatType>(calculatedObjectives, front,
-                                           extreme);
+    dimension = GetGeometry<MatType>(calculatedObjectives, front, extreme);
     for (size_t i = 0; i < front.size(); i++)
     {
       if (selected.count(i) == 0)
@@ -758,17 +787,17 @@ inline void AGEMOEA::SurvivalScoreAssignment(
     }
 
     arma::Mat<ElemType> pairwise(front.size(), front.size(), arma::fill::zeros);
-    PairwiseDistance<MatType>(pairwise,calculatedObjectives,front,dimension);
-    arma::Row<typename MatType::elem_type> value(front.size(), 
+    PairwiseDistance<MatType>(pairwise, calculatedObjectives, front, dimension);
+    arma::Row<typename MatType::elem_type> value(front.size(),
         arma::fill::zeros);
-    
+
     // Calculate the diversity and proximity score.
     for (size_t i = 0; i < front.size(); i++)
     {
-      pairwise.col(i) = pairwise.col(i) / std::pow(arma::accu(arma::pow(
-        arma::abs(calculatedObjectives[front[i]]), dimension)), 1.0 / dimension);
+      pairwise.col(i) = pairwise.col(i) / std::pow(accu(pow(
+        arma::abs(calculatedObjectives[front[i]]), dimension)), 1 / dimension);
     }
-    
+
     while (remaining.size() > 0)
     {
       std::set<size_t>::iterator it;
@@ -789,12 +818,12 @@ inline void AGEMOEA::SurvivalScoreAssignment(
   {
     for (size_t i = 0; i < front.size(); i++)
     {
-      calculatedObjectives[front[i]] = (calculatedObjectives[front[i]]) / normalize;
-      survivalScore[front[i]] =  1.0 / std::pow(arma::accu(arma::pow(arma::abs(
-          calculatedObjectives[front[i]] - idealPoint), dimension)), 
-              1.0 / dimension);
+      calculatedObjectives[front[i]] =
+          (calculatedObjectives[front[i]]) / normalize;
+      survivalScore[front[i]] =  1 / std::pow(accu(pow(abs(
+          calculatedObjectives[front[i]] - idealPoint), dimension)),
+              1 / dimension);
     }
-
   }
 }
 

include/ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp

@@ -30,7 +30,8 @@ namespace ens {
  * documentation on function types included with this distribution or on the
  * ensmallen website.
  */
-class AugLagrangian
+template<typename VecType = arma::vec> // TODO: remove for ensmallen 4.x
+class AugLagrangianType
 {
  public:
   /**
@@ -43,13 +44,13 @@ class AugLagrangian
    * @param maxIterations Maximum number of iterations of the Augmented
    *     Lagrangian algorithm.  0 indicates no maximum.
    */
-  AugLagrangian(const size_t maxIterations = 1000,
-                const double penaltyThresholdFactor = 0.25,
-                const double sigmaUpdateFactor = 10.0,
-                const L_BFGS& lbfgs = L_BFGS());
+  AugLagrangianType(const size_t maxIterations = 1000,
+                    const double penaltyThresholdFactor = 0.25,
+                    const double sigmaUpdateFactor = 10.0,
+                    const L_BFGS& lbfgs = L_BFGS());
 
   /**
-   * Optimize the function.  The value '1' is used for the initial value of each
+   * Optimize the function.  The value '0' is used for the initial value of each
    * Lagrange multiplier.  To set the Lagrange multipliers yourself, use the
    * other overload of Optimize().
    *
@@ -66,7 +67,8 @@ class AugLagrangian
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  typename std::enable_if<IsMatrixType<GradType>::value &&
+                          IsAllNonMatrix<CallbackTypes...>::value, bool>::type
   Optimize(LagrangianFunctionType& function,
            MatType& coordinates,
            CallbackTypes&&... callbacks);
@@ -75,9 +77,10 @@ class AugLagrangian
   template<typename LagrangianFunctionType,
            typename MatType,
            typename... CallbackTypes>
-  bool Optimize(LagrangianFunctionType& function,
-                MatType& coordinates,
-                CallbackTypes&&... callbacks)
+  typename std::enable_if<IsAllNonMatrix<CallbackTypes...>::value, bool>::type
+  Optimize(LagrangianFunctionType& function,
+           MatType& coordinates,
+           CallbackTypes&&... callbacks)
   {
     return Optimize<LagrangianFunctionType, MatType, MatType,
         CallbackTypes...>(function, coordinates,
@@ -96,29 +99,53 @@ class AugLagrangian
    * @tparam CallbackTypes Types of callback functions.
    * @param function The function to optimize.
    * @param coordinates Output matrix to store the optimized coordinates in.
-   * @param initLambda Vector of initial Lagrange multipliers.  Should have
-   *     length equal to the number of constraints.
-   * @param initSigma Initial penalty parameter.
+   * @param lambda Vector containing initial Lagrange multipliers.  Should have
+   *     length equal to the number of constraints.  This will be overwritten
+   *     with the Lagrange multipliers that are found during optimization.
+   * @param sigma Initial penalty parameter.  This will be overwritten with the
+   *     final penalty value used during optimization.
    * @param callbacks Callback functions.
    */
   template<typename LagrangianFunctionType,
            typename MatType,
+           typename InVecType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  [[deprecated("use Optimize() with non-const lambda/sigma instead")]]
+  typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
   Optimize(LagrangianFunctionType& function,
            MatType& coordinates,
-           const arma::vec& initLambda,
+           const InVecType& initLambda,
            const double initSigma,
+           CallbackTypes&&... callbacks)
+  {
+    deprecatedLambda = initLambda;
+    deprecatedSigma = initSigma;
+    return Optimize(function, coordinates, this->deprecatedLambda,
+        this->deprecatedSigma,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename InVecType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
+  Optimize(LagrangianFunctionType& function,
+           MatType& coordinates,
+           InVecType& lambda,
+           double& sigma,
            CallbackTypes&&... callbacks);
 
   //! Forward the MatType as GradType.
   template<typename LagrangianFunctionType,
            typename MatType,
            typename... CallbackTypes>
+  [[deprecated("use Optimize() with non-const lambda/sigma instead")]]
   bool Optimize(LagrangianFunctionType& function,
                 MatType& coordinates,
-                const arma::vec& initLambda,
+                const VecType& initLambda,
                 const double initSigma,
                 CallbackTypes&&... callbacks)
   {
@@ -127,20 +154,39 @@ class AugLagrangian
         std::forward<CallbackTypes>(callbacks)...);
   }
 
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename InVecType,
+           typename... CallbackTypes>
+  bool Optimize(LagrangianFunctionType& function,
+                MatType& coordinates,
+                InVecType& lambda,
+                double& sigma,
+                CallbackTypes&&... callbacks)
+  {
+    return Optimize<LagrangianFunctionType, MatType, InVecType, MatType,
+        CallbackTypes...>(function, coordinates, lambda, sigma,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
   //! Get the L-BFGS object used for the actual optimization.
   const L_BFGS& LBFGS() const { return lbfgs; }
   //! Modify the L-BFGS object used for the actual optimization.
   L_BFGS& LBFGS() { return lbfgs; }
 
   //! Get the Lagrange multipliers.
-  const arma::vec& Lambda() const { return lambda; }
+  [[deprecated("use Optimize() with lambda/sigma parameters instead")]]
+  const VecType& Lambda() const { return deprecatedLambda; }
   //! Modify the Lagrange multipliers (i.e. set them before optimization).
-  arma::vec& Lambda() { return lambda; }
+  [[deprecated("use Optimize() with lambda/sigma parameters instead")]]
+  VecType& Lambda() { return deprecatedLambda; }
 
   //! Get the penalty parameter.
-  double Sigma() const { return sigma; }
+  [[deprecated("use Optimize() with lambda/sigma parameters instead")]]
+  double Sigma() const { return deprecatedSigma; }
   //! Modify the penalty parameter.
-  double& Sigma() { return sigma; }
+  [[deprecated("use Optimize() with lambda/sigma parameters instead")]]
+  double& Sigma() { return deprecatedSigma; }
 
   //! Get the maximum iterations
   size_t MaxIterations() const { return maxIterations; }
@@ -173,11 +219,11 @@ class AugLagrangian
   //! Controls early termination of the optimization process.
   bool terminate;
 
+  // NOTE: these will be removed in ensmallen 4.x!
   //! Lagrange multipliers.
-  arma::vec lambda;
-
+  VecType deprecatedLambda;
   //! Penalty parameter.
-  double sigma;
+  double deprecatedSigma;
 
   /**
    * Internal optimization function: given an initialized AugLagrangianFunction,
@@ -185,27 +231,32 @@ class AugLagrangian
    */
   template<typename LagrangianFunctionType,
            typename MatType,
+           typename InVecType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
-  Optimize(AugLagrangianFunction<LagrangianFunctionType>& augfunc,
+  typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
+  Optimize(AugLagrangianFunction<LagrangianFunctionType, InVecType>& augfunc,
            MatType& coordinates,
            CallbackTypes&&... callbacks);
 
   //! Forward the MatType as GradType.
   template<typename LagrangianFunctionType,
            typename MatType,
+           typename InVecType,
            typename... CallbackTypes>
-  bool Optimize(AugLagrangianFunction<LagrangianFunctionType>& function,
-                MatType& coordinates,
-                CallbackTypes&&... callbacks)
+  bool Optimize(
+      AugLagrangianFunction<LagrangianFunctionType, InVecType>& function,
+      MatType& coordinates,
+      CallbackTypes&&... callbacks)
   {
-    return Optimize<LagrangianFunctionType, MatType, MatType,
+    return Optimize<LagrangianFunctionType, MatType, InVecType, MatType,
         CallbackTypes...>(function, coordinates,
         std::forward<CallbackTypes>(callbacks)...);
   }
 };
 
+using AugLagrangian = AugLagrangianType<arma::vec>;
+
 } // namespace ens
 
 #include "aug_lagrangian_impl.hpp"

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_function.hpp

@@ -31,19 +31,10 @@ namespace ens {
  *
  * @tparam LagrangianFunction Lagrangian function to be used.
  */
-template<typename LagrangianFunction>
+template<typename LagrangianFunction, typename VecType>
 class AugLagrangianFunction
 {
  public:
-  /**
-   * Initialize the AugLagrangianFunction, but don't set the Lagrange
-   * multipliers or penalty parameters yet.  Make sure you set the Lagrange
-   * multipliers before you use this...
-   *
-   * @param function Lagrangian function.
-   */
-  AugLagrangianFunction(LagrangianFunction& function);
-
   /**
    * Initialize the AugLagrangianFunction with the given LagrangianFunction,
    * Lagrange multipliers, and initial penalty parameter.
@@ -53,8 +44,8 @@ class AugLagrangianFunction
    * @param sigma Initial penalty parameter.
    */
   AugLagrangianFunction(LagrangianFunction& function,
-                        const arma::vec& lambda,
-                        const double sigma);
+                        VecType& lambda,
+                        double& sigma);
   /**
    * Evaluate the objective function of the Augmented Lagrangian function, which
    * is the standard Lagrangian function evaluation plus a penalty term, which
@@ -81,17 +72,12 @@ class AugLagrangianFunction
    *
    * @return Initial point.
    */
-  template<typename MatType = arma::mat>
+  template<typename MatType>
   const MatType& GetInitialPoint() const;
 
-  //! Get the Lagrange multipliers.
-  const arma::vec& Lambda() const { return lambda; }
-  //! Modify the Lagrange multipliers.
-  arma::vec& Lambda() { return lambda; }
-
-  //! Get sigma (the penalty parameter).
-  double Sigma() const { return sigma; }
-  //! Modify sigma (the penalty parameter).
+  // Get the Lagrange multipliers.
+  VecType& Lambda() { return lambda; }
+  // Get the penalty parameter.
   double& Sigma() { return sigma; }
 
   //! Get the Lagrangian function.
@@ -104,9 +90,9 @@ class AugLagrangianFunction
   LagrangianFunction& function;
 
   //! The Lagrange multipliers.
-  arma::vec lambda;
+  VecType& lambda;
   //! The penalty parameter.
-  double sigma;
+  double& sigma;
 };
 
 } // namespace ens

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_function_impl.hpp

@@ -20,23 +20,11 @@
 namespace ens {
 
 // Initialize the AugLagrangianFunction.
-template<typename LagrangianFunction>
-AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
-    LagrangianFunction& function) :
-    function(function),
-    lambda(function.NumConstraints()),
-    sigma(10)
-{
-  // Initialize lambda vector to all zeroes.
-  lambda.zeros();
-}
-
-// Initialize the AugLagrangianFunction.
-template<typename LagrangianFunction>
-AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
+template<typename LagrangianFunction, typename VecType>
+AugLagrangianFunction<LagrangianFunction, VecType>::AugLagrangianFunction(
     LagrangianFunction& function,
-    const arma::vec& lambda,
-    const double sigma) :
+    VecType& lambda,
+    double& sigma) :
     function(function),
     lambda(lambda),
     sigma(sigma)
@@ -45,9 +33,10 @@ AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
 }
 
 // Evaluate the AugLagrangianFunction at the given coordinates.
-template<typename LagrangianFunction>
+template<typename LagrangianFunction, typename VecType>
 template<typename MatType>
-typename MatType::elem_type AugLagrangianFunction<LagrangianFunction>::Evaluate(
+typename MatType::elem_type
+AugLagrangianFunction<LagrangianFunction, VecType>::Evaluate(
     const MatType& coordinates) const
 {
   // The augmented Lagrangian is evaluated as
@@ -63,20 +52,22 @@ typename MatType::elem_type AugLagrangianFunction<LagrangianFunction>::Evaluate(
   {
     ElemType constraint = function.EvaluateConstraint(i, coordinates);
 
-    objective += (-lambda[i] * constraint) +
-        sigma * std::pow(constraint, 2) / 2;
+    objective += (-ElemType(lambda[i]) * constraint) +
+        ElemType(sigma) * std::pow(constraint, ElemType(2)) / 2;
   }
 
   return objective;
 }
 
 // Evaluate the gradient of the AugLagrangianFunction at the given coordinates.
-template<typename LagrangianFunction>
+template<typename LagrangianFunction, typename VecType>
 template<typename MatType, typename GradType>
-void AugLagrangianFunction<LagrangianFunction>::Gradient(
+void AugLagrangianFunction<LagrangianFunction, VecType>::Gradient(
     const MatType& coordinates,
     GradType& gradient) const
 {
+  typedef typename MatType::elem_type ElemType;
+
   // The augmented Lagrangian's gradient is evaluted as
   // f'(x) + {(-lambda_i + sigma * c_i(x)) * c'_i(x)} for all constraints
   gradient.zeros();
@@ -89,16 +80,17 @@ void AugLagrangianFunction<LagrangianFunction>::Gradient(
 
     // Now calculate scaling factor and add to existing gradient.
     GradType tmpGradient;
-    tmpGradient = (-lambda[i] + sigma *
+    tmpGradient = (ElemType(-lambda[i]) + ElemType(sigma) *
         function.EvaluateConstraint(i, coordinates)) * constraintGradient;
     gradient += tmpGradient;
   }
 }
 
 // Get the initial point.
-template<typename LagrangianFunction>
+template<typename LagrangianFunction, typename VecType>
 template<typename MatType>
-const MatType& AugLagrangianFunction<LagrangianFunction>::GetInitialPoint()
+const MatType&
+AugLagrangianFunction<LagrangianFunction, VecType>::GetInitialPoint()
     const
 {
   return function.template GetInitialPoint<MatType>();

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_impl.hpp

@@ -19,70 +19,90 @@
 
 namespace ens {
 
-inline AugLagrangian::AugLagrangian(const size_t maxIterations,
-                                    const double penaltyThresholdFactor,
-                                    const double sigmaUpdateFactor,
-                                    const L_BFGS& lbfgs) :
+template<typename VecType>
+inline AugLagrangianType<VecType>::AugLagrangianType(
+    const size_t maxIterations,
+    const double penaltyThresholdFactor,
+    const double sigmaUpdateFactor,
+    const L_BFGS& lbfgs) :
     maxIterations(maxIterations),
     penaltyThresholdFactor(penaltyThresholdFactor),
     sigmaUpdateFactor(sigmaUpdateFactor),
     lbfgs(lbfgs),
     terminate(false),
-    sigma(0.0)
+    deprecatedSigma(0.0)
 {
 }
 
+template<typename VecType>
 template<typename LagrangianFunctionType,
          typename MatType,
+         typename InVecType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value, bool>::type
-AugLagrangian::Optimize(LagrangianFunctionType& function,
-                        MatType& coordinates,
-                        const arma::vec& initLambda,
-                        const double initSigma,
-                        CallbackTypes&&... callbacks)
+typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
+AugLagrangianType<VecType>::Optimize(
+    LagrangianFunctionType& function,
+    MatType& coordinates,
+    InVecType& lambda,
+    double& sigma,
+    CallbackTypes&&... callbacks)
 {
-  lambda = initLambda;
-  sigma = initSigma;
-
-  AugLagrangianFunction<LagrangianFunctionType> augfunc(function,
-      lambda, sigma);
+  AugLagrangianFunction<LagrangianFunctionType, InVecType> augfunc(
+      function, lambda, sigma);
 
   return Optimize(augfunc, coordinates, callbacks...);
 }
 
+template<typename VecType>
 template<typename LagrangianFunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value, bool>::type
-AugLagrangian::Optimize(LagrangianFunctionType& function,
-                        MatType& coordinates,
-                        CallbackTypes&&... callbacks)
+typename std::enable_if<IsMatrixType<GradType>::value &&
+                        IsAllNonMatrix<CallbackTypes...>::value, bool>::type
+AugLagrangianType<VecType>::Optimize(LagrangianFunctionType& function,
+                                     MatType& coordinates,
+                                     CallbackTypes&&... callbacks)
 {
+  typedef typename ForwardType<MatType>::bvec InVecType;
+
   // If the user did not specify the right size for sigma and lambda, we will
   // use defaults.
-  if (!lambda.is_empty())
+  // TODO: remove this when ensmallen 4.x is released!
+  if (!deprecatedLambda.is_empty())
   {
-    AugLagrangianFunction<LagrangianFunctionType> augfunc(function, lambda,
-        sigma);
-    return Optimize(augfunc, coordinates, callbacks...);
+    InVecType lambda(conv_to<InVecType>::from(deprecatedLambda));
+
+    AugLagrangianFunction<LagrangianFunctionType, InVecType> augfunc(function,
+        lambda, deprecatedSigma);
+    const bool result = Optimize(augfunc, coordinates, callbacks...);
+    deprecatedLambda = conv_to<VecType>::from(lambda);
+
+    return result;
   }
   else
   {
-    AugLagrangianFunction<LagrangianFunctionType> augfunc(function);
+    // Use default values.
+    InVecType lambda(function.NumConstraints());
+    lambda.zeros();
+    double sigma = 10;
+
+    AugLagrangianFunction<LagrangianFunctionType, InVecType> augfunc(
+        function, lambda, sigma);
     return Optimize(augfunc, coordinates, callbacks...);
   }
 }
 
+template<typename VecType>
 template<typename LagrangianFunctionType,
          typename MatType,
+         typename InVecType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value, bool>::type
-AugLagrangian::Optimize(
-    AugLagrangianFunction<LagrangianFunctionType>& augfunc,
+typename std::enable_if<IsMatrixType<GradType>::value, bool>::type
+AugLagrangianType<VecType>::Optimize(
+    AugLagrangianFunction<LagrangianFunctionType, InVecType>& augfunc,
     MatType& coordinatesIn,
     CallbackTypes&&... callbacks)
 {
@@ -110,13 +130,14 @@ AugLagrangian::Optimize(
 
   // Convergence tolerance---depends on the epsilon of the type we are using for
   // optimization.
-  ElemType tolerance = 1e3 * std::numeric_limits<ElemType>::epsilon();
+  ElemType tolerance = 1000 * std::numeric_limits<ElemType>::epsilon();
 
   // Then, calculate the current penalty.
   ElemType penalty = 0;
   for (size_t i = 0; i < function.NumConstraints(); i++)
   {
-    const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates), 2);
+    const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates),
+        ElemType(2));
     terminate |= Callback::EvaluateConstraint(*this, function, coordinates, i,
         p, callbacks...);
 
@@ -149,9 +170,6 @@ AugLagrangian::Optimize(
     if (std::abs(lastObjective - objective) < tolerance &&
         augfunc.Sigma() > 500000)
     {
-      lambda = std::move(augfunc.Lambda());
-      sigma = augfunc.Sigma();
-
       Callback::EndOptimization(*this, function, coordinates, callbacks...);
       return true;
     }
@@ -167,7 +185,7 @@ AugLagrangian::Optimize(
     for (size_t i = 0; i < function.NumConstraints(); i++)
     {
       const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates),
-          2);
+          ElemType(2));
       terminate |= Callback::EvaluateConstraint(*this, function, coordinates, i,
           p, callbacks...);
 
@@ -190,12 +208,12 @@ AugLagrangian::Optimize(
         terminate |= Callback::EvaluateConstraint(*this, function, coordinates,
             i, p, callbacks...);
 
-        augfunc.Lambda()[i] -= augfunc.Sigma() * p;
+        augfunc.Lambda()[i] -= ElemType(augfunc.Sigma()) * p;
       }
 
       // We also update the penalty threshold to be a factor of the current
       // penalty.
-      penaltyThreshold = penaltyThresholdFactor * penalty;
+      penaltyThreshold = ElemType(penaltyThresholdFactor) * penalty;
       Info << "Lagrange multiplier estimates updated." << std::endl;
     }
     else
@@ -208,7 +226,7 @@ AugLagrangian::Optimize(
         Warn << "AugLagrangian::Optimize(): sigma too large for element type; "
             << "terminating." << std::endl;
         Callback::EndOptimization(*this, function, coordinates, callbacks...);
-        return false;
+        return true;
       }
     }
 

include/ensmallen_bits/bigbatch_sgd/adaptive_stepsize.hpp

@@ -69,6 +69,8 @@ class AdaptiveStepsize
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     // Create the instantiated object.
     Policy(AdaptiveStepsize& parent) : parent(parent) { }
 
@@ -104,7 +106,7 @@ class AdaptiveStepsize
           backtrackingBatchSize);
 
       // Update the iterate.
-      iterate -= stepSize * gradient;
+      iterate -= ElemType(stepSize) * gradient;
 
       // Update Gradient & calculate curvature of quadratic approximation.
       GradType functionGradient(iterate.n_rows, iterate.n_cols);
@@ -132,8 +134,8 @@ class AdaptiveStepsize
         delta0 = delta1 + (functionGradient - delta1) / k;
 
         // Compute sample variance.
-        vB += arma::norm(functionGradient - delta1, 2.0) *
-            arma::norm(functionGradient - delta0, 2.0);
+        vB += norm(functionGradient - delta1, 2.0) *
+            norm(functionGradient - delta0, 2.0);
 
         delta1 = delta0;
         gradient += functionGradient;
@@ -145,13 +147,13 @@ class AdaptiveStepsize
 
       // Update sample variance & norm of the gradient.
       sampleVariance = vB;
-      gradientNorm = std::pow(arma::norm(gradient / backtrackingBatchSize, 2),
+      gradientNorm = std::pow(norm(gradient / backtrackingBatchSize, 2),
           2.0);
 
       // Compute curvature.
-      double v = arma::trace(arma::trans(iterate - iteratePrev) *
+      double v = trace(trans(iterate - iteratePrev) *
           (gradient - gradPrevIterate)) /
-          std::pow(arma::norm(iterate - iteratePrev, 2), 2.0);
+          std::pow(norm(iterate - iteratePrev, 2), 2.0);
 
       // Update previous iterate.
       iteratePrev = iterate;
@@ -205,12 +207,10 @@ class AdaptiveStepsize
                       const size_t offset,
                       const size_t backtrackingBatchSize)
     {
-      typedef typename MatType::elem_type ElemType;
-
       ElemType overallObjective = function.Evaluate(iterate,
           offset, backtrackingBatchSize);
 
-      MatType iterateUpdate = iterate - (stepSize * gradient);
+      MatType iterateUpdate = iterate - (ElemType(stepSize) * gradient);
       ElemType overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
           backtrackingBatchSize);
 
@@ -220,7 +220,7 @@ class AdaptiveStepsize
       {
         stepSize *= parent.backtrackStepSize;
 
-        iterateUpdate = iterate - (stepSize * gradient);
+        iterateUpdate = iterate - (ElemType(stepSize) * gradient);
         overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
             backtrackingBatchSize);
       }

include/ensmallen_bits/bigbatch_sgd/backtracking_line_search.hpp

@@ -60,6 +60,8 @@ class BacktrackingLineSearch
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     // Instantiate the policy with the given parent.
     Policy(BacktrackingLineSearch& parent) : parent(parent) { }
 
@@ -94,12 +96,10 @@ class BacktrackingLineSearch
       if (reset)
         stepSize *= 2;
 
-      typedef typename MatType::elem_type ElemType;
-
       ElemType overallObjective = function.Evaluate(iterate, offset,
           backtrackingBatchSize);
 
-      MatType iterateUpdate = iterate - (stepSize * gradient);
+      MatType iterateUpdate = iterate - (ElemType(stepSize) * gradient);
       ElemType overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
           backtrackingBatchSize);
 
@@ -109,7 +109,7 @@ class BacktrackingLineSearch
       {
         stepSize /= 2;
 
-        iterateUpdate = iterate - (stepSize * gradient);
+        iterateUpdate = iterate - (ElemType(stepSize) * gradient);
         overallObjectiveUpdate = function.Evaluate(iterateUpdate,
           offset, backtrackingBatchSize);
       }

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp

@@ -125,7 +125,7 @@ class BigBatchSGD
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd_impl.hpp

@@ -50,8 +50,8 @@ template<typename SeparableFunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 BigBatchSGD<UpdatePolicyType>::Optimize(
     SeparableFunctionType& function,
     MatType& iterateIn,
@@ -137,13 +137,13 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
       delta0 = delta1 + (functionGradient - delta1) / k;
 
       // Compute sample variance.
-      vB += arma::norm(functionGradient - delta1, 2.0) *
-          arma::norm(functionGradient - delta0, 2.0);
+      vB += norm(functionGradient - delta1, 2.0) *
+          norm(functionGradient - delta0, 2.0);
 
       delta1 = delta0;
       gradient += functionGradient;
     }
-    double gB = std::pow(arma::norm(gradient / effectiveBatchSize, 2), 2.0);
+    double gB = std::pow(norm(gradient / effectiveBatchSize, 2), 2.0);
 
     // Reset the batch size update process counter.
     reset = false;
@@ -174,13 +174,13 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
           delta0 = delta1 + (functionGradient - delta1) / (k + 1);
 
           // Compute sample variance.
-          vB += arma::norm(functionGradient - delta1, 2.0) *
-              arma::norm(functionGradient - delta0, 2.0);
+          vB += norm(functionGradient - delta1, 2.0) *
+              norm(functionGradient - delta0, 2.0);
 
           delta1 = delta0;
           gradient += functionGradient;
         }
-        gB = std::pow(arma::norm(gradient / (batchSize + batchOffset), 2), 2.0);
+        gB = std::pow(norm(gradient / (batchSize + batchOffset), 2), 2.0);
 
         // Update the batchSize.
         batchSize += batchOffset;
@@ -199,7 +199,7 @@ BigBatchSGD<UpdatePolicyType>::Optimize(
         reset);
 
     // Update the iterate.
-    iterate -= stepSize * gradient;
+    iterate -= ElemType(stepSize) * gradient;
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
 
     const ElemType objective = f.Evaluate(iterate, currentFunction,

include/ensmallen_bits/callbacks/timer_stop.hpp

@@ -45,6 +45,30 @@ class TimerStop
     timer.tic();
   }
 
+  /**
+   * Callback function called when a step is taken.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */)
+  {
+    if (timer.toc() > duration)
+    {
+      Info << "Timer timeout (" << duration << "s) reached; terminating "
+          << "optimization." << std::endl;
+      return true;
+    }
+
+    return false;
+  }
+
   /**
    * Callback function called at the end of a pass over the data.
    *
@@ -63,7 +87,8 @@ class TimerStop
   {
     if (timer.toc() > duration)
     {
-      Info << "Timer timeout reached; terminate optimization." << std::endl;
+      Info << "Timer timeout (" << duration << "s) reached; terminating "
+          << "optimization." << std::endl;
       return true;
     }
 

include/ensmallen_bits/cd/cd.hpp

@@ -94,7 +94,7 @@ class CD
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(ResolvableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/cd/cd_impl.hpp

@@ -39,8 +39,8 @@ template <typename ResolvableFunctionType,
           typename MatType,
           typename GradType,
           typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 CD<DescentPolicyType>::Optimize(
     ResolvableFunctionType& function,
     MatType& iterateIn,
@@ -66,9 +66,12 @@ CD<DescentPolicyType>::Optimize(
   // Controls early termination of the optimization process.
   bool terminate = false;
 
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
   // Start iterating.
   Callback::BeginOptimization(*this, function, iterate, callbacks...);
-  for (size_t i = 1; i != maxIterations && !terminate; ++i)
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     // Get the coordinate to descend on.
     size_t featureIdx = descentPolicy.template DescentFeature<
@@ -84,7 +87,7 @@ CD<DescentPolicyType>::Optimize(
       break;
 
     // Update the decision variable with the partial gradient.
-    iterate.col(featureIdx) -= stepSize * gradient.col(featureIdx);
+    iterate.col(featureIdx) -= ElemType(stepSize) * gradient.col(featureIdx);
     terminate |= Callback::StepTaken(*this, function, iterate, callbacks...);
 
     // Check for convergence.
@@ -120,9 +123,12 @@ CD<DescentPolicyType>::Optimize(
     }
   }
 
-  Info << "CD: maximum iterations (" << maxIterations << ") reached; "
-      << "terminating optimization." << std::endl;
-
+  if (!terminate)
+  {
+    Info << "CD: maximum iterations (" << maxIterations << ") reached; "
+        << "terminating optimization." << std::endl;
+  }
+  
   // Calculate and return final objective.  No need to pay attention to the
   // result of the callback.
   const ElemType objective = function.Evaluate(iterate);

include/ensmallen_bits/cd/descent_policies/random_descent.hpp

@@ -52,8 +52,11 @@ class RandomDescent
                                const MatType& /* iterate */,
                                const ResolvableFunctionType& function)
   {
+    // return randi<size_t>(
+    //     arma::distr_param(0, function.NumFeatures() - 1));
+
     return arma::as_scalar(arma::randi<arma::uvec>(
-          1, arma::distr_param(0, function.NumFeatures() - 1)));
+      1, arma::distr_param(0, function.NumFeatures() - 1)));
   }
 };
 

include/ensmallen_bits/cmaes/active_cmaes.hpp

@@ -3,8 +3,8 @@
  * @author Marcus Edel
  * @author Suvarsha Chennareddy
  *
- * Definition of the Active Covariance Matrix Adaptation Evolution Strategy 
- * as proposed by G.A Jastrebski and D.V Arnold in "Improving Evolution 
+ * Definition of the Active Covariance Matrix Adaptation Evolution Strategy
+ * as proposed by G.A Jastrebski and D.V Arnold in "Improving Evolution
  * Strategies through Active Covariance Matrix Adaptation".
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -26,25 +26,25 @@ namespace ens {
  * Active CMA-ES is a variant of the stochastic search algorithm
  * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
  * Active CMA-ES actively reduces the uncertainty in unfavourable directions by
- * exploiting the information about bad mutations in the covariance matrix 
- * update step. This isn't for the purpose of accelerating progress, but 
- * instead for speeding up the adaptation of the covariance matrix (which, in 
+ * exploiting the information about bad mutations in the covariance matrix
+ * update step. This isn't for the purpose of accelerating progress, but
+ * instead for speeding up the adaptation of the covariance matrix (which, in
  * turn, will lead to faster progress).
  *
  * For more information, please refer to:
  *
  * @code
  * @INPROCEEDINGS{1688662,
- *   author={Jastrebski, G.A. and Arnold, D.V.},
- *   booktitle={2006 IEEE International Conference on Evolutionary 
-                Computation},
- *   title={Improving Evolution Strategies through Active Covariance 
-            Matrix Adaptation},
- *   year={2006},
- *   volume={},
- *   number={},
- *   pages={2814-2821},
- *   doi={10.1109/CEC.2006.1688662}}
+ *   author    = {Jastrebski, G.A. and Arnold, D.V.},
+ *   booktitle = {2006 IEEE International Conference on Evolutionary
+ *                Computation},
+ *   title     = {Improving Evolution Strategies through Active Covariance
+ *                Matrix Adaptation},
+ *   year      = {2006},
+ *   volume    = {},
+ *   number    = {},
+ *   pages     = {2814-2821},
+ *   doi       = {10.1109/CEC.2006.1688662}}
  * @endcode
  *
  * Active CMA-ES can optimize separable functions.  For more details, see the
@@ -52,7 +52,7 @@ namespace ens {
  * ensmallen website.
  *
  * @tparam SelectionPolicy The selection strategy used for the evaluation step.
- * @tparam TransformationPolicy The transformation strategy used to 
+ * @tparam TransformationPolicy The transformation strategy used to
  *       map decision variables to the desired domain during fitness evaluation
  *       and termination. Use EmptyTransformation if the domain isn't bounded.
  */
@@ -62,15 +62,15 @@ class ActiveCMAES
 {
  public:
   /**
-   * Construct the Active CMA-ES optimizer with the given function and parameters. The
-   * defaults here are not necessarily good for the given problem, so it is
-   * suggested that the values used be tailored to the task at hand.  The
-   * maximum number of iterations refers to the maximum number of points that
-   * are processed (i.e., one iteration equals one point; one iteration does not
-   * equal one pass over the dataset).
+   * Construct the Active CMA-ES optimizer with the given function and
+   * parameters. The defaults here are not necessarily good for the given
+   * problem, so it is suggested that the values used be tailored to the task at
+   * hand.  The maximum number of iterations refers to the maximum number of
+   * points that are processed (i.e., one iteration equals one point; one
+   * iteration does not equal one pass over the dataset).
    *
    * @param lambda The population size (0 use the default size).
-   * @param transformationPolicy Instantiated transformation policy used to 
+   * @param transformationPolicy Instantiated transformation policy used to
    *     map the coordinates to the desired domain.
    * @param batchSize Batch size to use for the objective calculation.
    * @param maxIterations Maximum number of iterations allowed (0 means no
@@ -82,7 +82,7 @@ class ActiveCMAES
    */
   ActiveCMAES(
       const size_t lambda = 0,
-      const TransformationPolicyType& 
+      const TransformationPolicyType&
           transformationPolicy = TransformationPolicyType(),
       const size_t batchSize = 32,
       const size_t maxIterations = 1000,
@@ -91,38 +91,9 @@ class ActiveCMAES
       double stepSize = 0);
 
   /**
-   * Construct the Active CMA-ES optimizer with the given function and parameters 
-   * (including lower and upper bounds). The defaults here are not necessarily 
-   * good for the given problem, so it is suggested that the values used be 
-   * tailored to the task at hand.  The maximum number of iterations refers to 
-   * the maximum number of points that are processed (i.e., one iteration 
-   * equals one point; one iteration does not equal one pass over the dataset). 
-   *
-   * @param lambda The population size(0 use the default size).
-   * @param lowerBound Lower bound of decision variables.
-   * @param upperBound Upper bound of decision variables.
-   * @param batchSize Batch size to use for the objective calculation.
-   * @param maxIterations Maximum number of iterations allowed(0 means no
-      limit).
-   * @param tolerance Maximum absolute tolerance to terminate algorithm.
-   * @param selectionPolicy Instantiated selection policy used to calculate the
-   * objective.
-   * @param stepSize Starting sigma/step size (will be modified).
-   */
-  ActiveCMAES(
-      const size_t lambda = 0,
-      const double lowerBound = -10,
-      const double upperBound = 10,
-      const size_t batchSize = 32,
-      const size_t maxIterations = 1000,
-      const double tolerance = 1e-5,
-      const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
-      double stepSize = 0);
-
-  /**
-   * Optimize the given function using Active CMA-ES. The given starting point will be
-   * modified to store the finishing point of the algorithm, and the final
-   * objective value is returned.
+   * Optimize the given function using Active CMA-ES. The given starting point
+   * will be modified to store the finishing point of the algorithm, and the
+   * final objective value is returned.
    *
    * @tparam SeparableFunctionType Type of the function to be optimized.
    * @tparam MatType Type of matrix to optimize.
@@ -169,7 +140,7 @@ class ActiveCMAES
   const TransformationPolicyType& TransformationPolicy() const
   { return transformationPolicy; }
   //! Modify the transformation policy.
-  TransformationPolicyType& TransformationPolicy() 
+  TransformationPolicyType& TransformationPolicy()
   { return transformationPolicy; }
 
   //! Get the step size.
@@ -196,7 +167,7 @@ class ActiveCMAES
   SelectionPolicyType selectionPolicy;
 
   //! The transformationPolicy used to map coordinates to the suitable domain
-  //! while evaluating fitness. This mapping is also done after optimization 
+  //! while evaluating fitness. This mapping is also done after optimization
   //! has completed.
   TransformationPolicyType transformationPolicy;
 

include/ensmallen_bits/cmaes/active_cmaes_impl.hpp

@@ -18,7 +18,6 @@
 // In case it hasn't been included yet.
 #include "active_cmaes.hpp"
 
-#include "not_empty_transformation.hpp"
 #include <ensmallen_bits/function.hpp>
 
 namespace ens {
@@ -42,29 +41,6 @@ ActiveCMAES<SelectionPolicyType, TransformationPolicyType>::ActiveCMAES(
     stepSize(stepSizeIn)
 { /* Nothing to do. */ }
 
-template<typename SelectionPolicyType, typename TransformationPolicyType>
-ActiveCMAES<SelectionPolicyType, TransformationPolicyType>::ActiveCMAES(
-                                  const size_t lambda,
-                                  const double lowerBound,
-                                  const double upperBound,
-                                  const size_t batchSize,
-                                  const size_t maxIterations,
-                                  const double tolerance,
-                                  const SelectionPolicyType& selectionPolicy,
-                                  double stepSizeIn) :
-    lambda(lambda),
-    batchSize(batchSize),
-    maxIterations(maxIterations),
-    tolerance(tolerance),
-    selectionPolicy(selectionPolicy),
-    stepSize(stepSizeIn)
-{
-  Warn << "This is a deprecated constructor and will be removed in a "
-    "future version of ensmallen" << std::endl;
-  NotEmptyTransformation<TransformationPolicyType, EmptyTransformation<>> d;
-  d.Assign(transformationPolicy, lowerBound, upperBound);
-}
-
 //! Optimize the function (minimize).
 template<typename SelectionPolicyType, typename TransformationPolicyType>
 template<typename SeparableFunctionType,
@@ -80,6 +56,9 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
   typedef typename MatType::elem_type ElemType;
   typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
 
+  typedef typename ForwardType<MatType>::bcol BaseColType;
+  typedef typename ForwardType<MatType>::uvec UVecType;
+
   // Make sure that we have the methods that we need.  Long name...
   traits::CheckArbitrarySeparableFunctionTypeAPI<
       SeparableFunctionType, BaseMatType>();
@@ -105,21 +84,23 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
 
   // Step size control parameters.
   BaseMatType sigma(2, 1); // sigma is vector-shaped.
-  if (stepSize == 0) 
+  if (stepSize == 0)
     sigma(0) = transformationPolicy.InitialStepSize();
-  else 
-    sigma(0) = stepSize;
+  else
+    sigma(0) = ElemType(stepSize);
 
-  const ElemType cs = 4.0 / (iterate.n_elem + 4);
+  const ElemType cs = 4 / ElemType(iterate.n_elem + 4);
   const ElemType ds = 1 + cs;
-  const ElemType enn = std::sqrt(iterate.n_elem) * (1.0 - 1.0 /
-      (4.0 * iterate.n_elem) + 1.0 / (21 * std::pow(iterate.n_elem, 2)));
+  const ElemType enn = std::sqrt(iterate.n_elem) * (1 -
+      1 / ElemType(4 * iterate.n_elem) +
+      1 / ElemType(21 * std::pow(iterate.n_elem, 2)));
 
   // Covariance update parameters. Cumulation for distribution.
   const ElemType cc = cs;
-  const ElemType ccov = 2.0 / std::pow((iterate.n_elem + std::sqrt(2)), 2);
-  const ElemType beta = (4.0 * mu - 2.0) / (std::pow((iterate.n_elem + 12), 2) 
-      + 4 * mu);
+  const ElemType ccov = 2 /
+      std::pow((iterate.n_elem + std::sqrt(ElemType(2))), ElemType(2));
+  const ElemType beta = (4 * mu - 2) /
+      (std::pow(ElemType(iterate.n_elem + 12), ElemType(2)) + 4 * mu);
 
   std::vector<BaseMatType> mPosition(2, BaseMatType(iterate.n_rows,
       iterate.n_cols));
@@ -163,13 +144,16 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
   C[0].eye();
 
   // Covariance matrix parameters.
-  arma::Col<ElemType> eigval;
+  BaseColType eigval;
   BaseMatType eigvec;
   BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped.
   eigvalZero.zeros();
 
   // The current visitation order (sorted by population objectives).
-  arma::uvec idx = arma::linspace<arma::uvec>(0, lambda - 1, lambda);
+  UVecType idx = linspace<UVecType>(0, lambda - 1, lambda);
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
 
   // Now iterate!
   Callback::BeginOptimization(*this, function, transformedIterate,
@@ -182,30 +166,31 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
   size_t patience = 10 + (30 * iterate.n_elem / lambda) + 1;
   size_t steps = 0;
 
-  for (size_t i = 1; (i != maxIterations) && !terminate; ++i)
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     // To keep track of where we are.
-    idx0 = (i - 1) % 2;
-    idx1 = i % 2;
+    idx0 = i % 2;
+    idx1 = (i + 1) % 2;
 
     // Perform Cholesky decomposition. If the matrix is not positive definite,
     // add a small value and try again.
     BaseMatType covLower;
-    while (!arma::chol(covLower, C[idx0], "lower"))
+    // while (!arma::chol(covLower, C[idx0], "lower"))
+    while (!chol(covLower, C[idx0]))
       C[idx0].diag() += std::numeric_limits<ElemType>::epsilon();
 
-    arma::eig_sym(eigval, eigvec, C[idx0]);
+    eig_sym(eigval, eigvec, C[idx0]);
 
     for (size_t j = 0; j < lambda; ++j)
     {
       if (iterate.n_rows > iterate.n_cols)
       {
         pStep[idx(j)] = covLower *
-          arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
+          randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
       }
       else
       {
-        pStep[idx(j)] = arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
+        pStep[idx(j)] = randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
           * covLower.t();
       }
 
@@ -218,7 +203,7 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
     }
 
     // Sort population.
-    idx = arma::sort_index(pObjective);
+    idx = sort_index(pObjective);
 
     step = w * pStep[idx(0)];
     for (size_t j = 1; j < mu; ++j)
@@ -256,7 +241,7 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
           eigvec * diagmat(1 / eigval) * eigvec.t();
     }
 
-    const ElemType psNorm = arma::norm(ps[idx1]);
+    const ElemType psNorm = norm(ps[idx1]);
     sigma(idx1) = sigma(idx0) * std::exp(cs / ds * (psNorm / enn - 1));
 
     if (std::isnan(sigma(idx1)) || sigma(idx1) > 1e14)
@@ -308,8 +293,8 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
       }
     }
 
-    arma::eig_sym(eigval, eigvec, C[idx1]);
-    const arma::uvec negativeEigval = arma::find(eigval < 0, 1);
+    eig_sym(eigval, eigvec, C[idx1]);
+    const UVecType negativeEigval = find(eigval < 0, 1);
     if (!negativeEigval.is_empty())
     {
       if (negativeEigval(0) == 0)
@@ -319,7 +304,7 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
       else
       {
         C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) *
-            arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
+            diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
             eigvec.cols(0, negativeEigval(0) - 1).t();
       }
     }

include/ensmallen_bits/cmaes/cmaes.hpp

@@ -48,7 +48,7 @@ namespace ens {
  * ensmallen website.
  *
  * @tparam SelectionPolicy The selection strategy used for the evaluation step.
- * @tparam TransformationPolicy The transformation strategy used to 
+ * @tparam TransformationPolicy The transformation strategy used to
  *       map decision variables to the desired domain during fitness evaluation
  *       and termination. Use EmptyTransformation if the domain isn't bounded.
  */
@@ -66,7 +66,7 @@ class CMAES
    * equal one pass over the dataset).
    *
    * @param lambda The population size (0 use the default size).
-   * @param transformationPolicy Instantiated transformation policy used to 
+   * @param transformationPolicy Instantiated transformation policy used to
    *     map the coordinates to the desired domain.
    * @param batchSize Batch size to use for the objective calculation.
    * @param maxIterations Maximum number of iterations allowed (0 means no
@@ -77,7 +77,7 @@ class CMAES
    * @param stepSize Starting sigma/step size (will be modified).
    */
   CMAES(const size_t lambda = 0,
-        const TransformationPolicyType& 
+        const TransformationPolicyType&
               transformationPolicy = TransformationPolicyType(),
         const size_t batchSize = 32,
         const size_t maxIterations = 1000,
@@ -85,34 +85,6 @@ class CMAES
         const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
         double stepSize = 0);
 
-  /**
-   * Construct the CMA-ES optimizer with the given function and parameters 
-   * (including lower and upper bounds). The defaults here are not necessarily 
-   * good for the given problem, so it is suggested that the values used be 
-   * tailored to the task at hand.  The maximum number of iterations refers to 
-   * the maximum number of points that are processed (i.e., one iteration 
-   * equals one point; one iteration does not equal one pass over the dataset).
-   *
-   * @param lambda The population size(0 use the default size).
-   * @param lowerBound Lower bound of decision variables.
-   * @param upperBound Upper bound of decision variables.
-   * @param batchSize Batch size to use for the objective calculation.
-   * @param maxIterations Maximum number of iterations allowed(0 means no
-      limit).
-   * @param tolerance Maximum absolute tolerance to terminate algorithm.
-   * @param selectionPolicy Instantiated selection policy used to calculate the
-   * objective.
-   * @param stepSize Starting sigma/step size (will be modified).
-   */
-  CMAES(const size_t lambda = 0,
-        const double lowerBound = -10,
-        const double upperBound = 10,
-        const size_t batchSize = 32,
-        const size_t maxIterations = 1000,
-        const double tolerance = 1e-5,
-        const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
-        double stepSize = 0);
-
   /**
    * Optimize the given function using CMA-ES. The given starting point will be
    * modified to store the finishing point of the algorithm, and the final
@@ -162,15 +134,13 @@ class CMAES
   const TransformationPolicyType& TransformationPolicy() const
   { return transformationPolicy; }
   //! Modify the transformation policy.
-  TransformationPolicyType& TransformationPolicy() 
+  TransformationPolicyType& TransformationPolicy()
   { return transformationPolicy; }
 
   //! Get the step size.
-  double StepSize() const
-  { return stepSize; }
+  double StepSize() const { return stepSize; }
   //! Modify the step size.
-  double& StepSize()
-  { return stepSize; }
+  double& StepSize() { return stepSize; }
 
   //! Get the total number of function evaluations.
   size_t FunctionEvaluations() const  { return functionEvaluations; }
@@ -192,7 +162,7 @@ class CMAES
   SelectionPolicyType selectionPolicy;
 
   //! The transformationPolicy used to map coordinates to the suitable domain
-  //! while evaluating fitness. This mapping is also done after optimization 
+  //! while evaluating fitness. This mapping is also done after optimization
   //! has completed.
   TransformationPolicyType transformationPolicy;
 

include/ensmallen_bits/cmaes/cmaes_impl.hpp

@@ -18,14 +18,13 @@
 // In case it hasn't been included yet.
 #include "cmaes.hpp"
 
-#include "not_empty_transformation.hpp"
 #include <ensmallen_bits/function.hpp>
 
 namespace ens {
 
 template<typename SelectionPolicyType, typename TransformationPolicyType>
 CMAES<SelectionPolicyType, TransformationPolicyType>::CMAES(const size_t lambda,
-                                  const TransformationPolicyType& 
+                                  const TransformationPolicyType&
                                         transformationPolicy,
                                   const size_t batchSize,
                                   const size_t maxIterations,
@@ -41,35 +40,12 @@ CMAES<SelectionPolicyType, TransformationPolicyType>::CMAES(const size_t lambda,
     stepSize(stepSizeIn)
 { /* Nothing to do. */ }
 
-template<typename SelectionPolicyType, typename TransformationPolicyType>
-CMAES<SelectionPolicyType, TransformationPolicyType>::CMAES(const size_t lambda,
-                                  const double lowerBound,
-                                  const double upperBound,
-                                  const size_t batchSize,
-                                  const size_t maxIterations,
-                                  const double tolerance,
-                                  const SelectionPolicyType& selectionPolicy,
-                                  double stepSizeIn) :
-    lambda(lambda),
-    batchSize(batchSize),
-    maxIterations(maxIterations),
-    tolerance(tolerance),
-    selectionPolicy(selectionPolicy),
-    stepSize(stepSizeIn)
-{
-  Warn << "This is a deprecated constructor and will be removed in a "
-    "future version of ensmallen" << std::endl;
-  NotEmptyTransformation<TransformationPolicyType, EmptyTransformation<>> d;
-  d.Assign(transformationPolicy, lowerBound, upperBound);
-}
-
-
 //! Optimize the function (minimize).
 template<typename SelectionPolicyType, typename TransformationPolicyType>
 template<typename SeparableFunctionType,
          typename MatType,
          typename... CallbackTypes>
-typename MatType::elem_type CMAES<SelectionPolicyType, 
+typename MatType::elem_type CMAES<SelectionPolicyType,
   TransformationPolicyType>::Optimize(
     SeparableFunctionType& function,
     MatType& iterateIn,
@@ -77,7 +53,10 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
 {
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
-  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  typedef typename ForwardType<MatType>::bcol bcol;
+  typedef typename ForwardType<MatType>::uvec UVecType;
+  typedef typename ForwardType<MatType>::bmat BaseMatType;
 
   // Make sure that we have the methods that we need.  Long name...
   traits::CheckArbitrarySeparableFunctionTypeAPI<
@@ -95,18 +74,18 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
 
   // Parent weights.
   const size_t mu = std::round(lambda / 2);
-  BaseMatType w = std::log(mu + 0.5) - arma::log(
-      arma::linspace<BaseMatType>(0, mu - 1, mu) + 1.0);
-  w /= arma::accu(w);
+  BaseMatType w = std::log(mu + 0.5) - log(
+      linspace<BaseMatType>(0, mu - 1, mu) + 1.0);
+  w /= accu(w);
 
   // Number of effective solutions.
-  const double muEffective = 1 / arma::accu(arma::pow(w, 2));
+  const double muEffective = 1 / accu(pow(w, 2));
 
   // Step size control parameters.
   BaseMatType sigma(2, 1); // sigma is vector-shaped.
-  if (stepSize == 0) 
+  if (stepSize == 0)
     sigma(0) = transformationPolicy.InitialStepSize();
-  else 
+  else
     sigma(0) = stepSize;
 
   const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5);
@@ -151,7 +130,6 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
     terminate |= Callback::Evaluate(*this, function, transformedIterate,
         objective, callbacks...);
   }
-  functionEvaluations += numFunctions;
 
   ElemType overallObjective = currentObjective;
   ElemType lastObjective = std::numeric_limits<ElemType>::max();
@@ -170,13 +148,16 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
   C[0].eye();
 
   // Covariance matrix parameters.
-  arma::Col<ElemType> eigval; // TODO: might need a more general type.
+  bcol eigval; // TODO: might need a more general type.
   BaseMatType eigvec;
   BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped.
   eigvalZero.zeros();
 
   // The current visitation order (sorted by population objectives).
-  arma::uvec idx = arma::linspace<arma::uvec>(0, lambda - 1, lambda);
+  UVecType idx = linspace<UVecType>(0, lambda - 1, lambda);
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
 
   // Now iterate!
   Callback::BeginOptimization(*this, function, transformedIterate,
@@ -187,31 +168,33 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
   size_t patience = 10 + (30 * iterate.n_elem / lambda) + 1;
   size_t steps = 0;
 
-  for (size_t i = 1; (i != maxIterations) && !terminate; ++i)
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     // To keep track of where we are.
-    const size_t idx0 = (i - 1) % 2;
-    const size_t idx1 = i % 2;
+    const size_t idx0 = i % 2;
+    const size_t idx1 = (i + 1) % 2;
 
     // Perform Cholesky decomposition. If the matrix is not positive definite,
     // add a small value and try again.
     BaseMatType covLower;
-    while (!arma::chol(covLower, C[idx0], "lower"))
+    // while (!chol(covLower, C[idx0], "lower"))
+    while (!chol(covLower, C[idx0]))
       C[idx0].diag() += std::numeric_limits<ElemType>::epsilon();
 
-    arma::eig_sym(eigval, eigvec, C[idx0]);
+    eig_sym(eigval, eigvec, C[idx0]);
 
     for (size_t j = 0; j < lambda; ++j)
     {
       if (iterate.n_rows > iterate.n_cols)
       {
-        pStep[idx(j)] = covLower *
-          arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
+        pStep[idx(j)] = covLower * BaseMatType(
+            iterate.n_rows, iterate.n_cols, GetFillType<MatType>::randn);
       }
       else
       {
-        pStep[idx(j)] = arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
-          * covLower.t();
+        pStep[idx(j)] = BaseMatType(
+          iterate.n_rows, iterate.n_cols, GetFillType<MatType>::randn) *
+          covLower.t();
       }
 
       pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];
@@ -223,7 +206,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
     }
 
     // Sort population.
-    idx = arma::sort_index(pObjective);
+    idx = sort_index(pObjective);
 
     step = w(0) * pStep[idx(0)];
     for (size_t j = 1; j < mu; ++j)
@@ -236,8 +219,6 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
         transformationPolicy.Transform(mPosition[idx1]), terminate,
         callbacks...);
 
-    functionEvaluations += lambda; 
-
     // Update best parameters.
     if (currentObjective < overallObjective)
     {
@@ -246,30 +227,30 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
 
       transformedIterate = transformationPolicy.Transform(iterate);
       terminate |= Callback::StepTaken(*this, function,
-        transformedIterate, callbacks...);
+          transformedIterate, callbacks...);
     }
 
     // Update Step Size.
     if (iterate.n_rows > iterate.n_cols)
     {
       ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
-        cs * (2 - cs) * muEffective) *
-        eigvec * diagmat(1 / eigval) * eigvec.t() * step;
+          cs * (2 - cs) * muEffective) * eigvec *
+          diagmat(1 / eigval) * eigvec.t() * step;
     }
     else
     {
       ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
-        cs * (2 - cs) * muEffective) * step *
-        eigvec * diagmat(1 / eigval) * eigvec.t();
+          cs * (2 - cs) * muEffective) * step * eigvec *
+          diagmat(1 / eigval) * eigvec.t();
     }
 
-    const ElemType psNorm = arma::norm(ps[idx1]);
+    const ElemType psNorm = norm(ps[idx1]);
     sigma(idx1) = sigma(idx0) * std::exp(cs / ds * (psNorm / enn - 1));
 
     if (std::isnan(sigma(idx1)) || sigma(idx1) > 1e14)
     {
       Warn << "The step size diverged to " << sigma(idx1) << "; "
-        << "terminating with failure.  Try a smaller step size?" << std::endl;
+          << "terminating with failure.  Try a smaller step size?" << std::endl;
 
       iterate = transformationPolicy.Transform(iterate);
 
@@ -278,20 +259,20 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
     }
 
     // Update covariance matrix.
-    if ((psNorm / sqrt(1 - std::pow(1 - cs, 2 * i))) < h)
+    if ((psNorm / std::sqrt(1 - std::pow(1.0 - cs, 2.0 * (double) i))) < h)
     {
       pc[idx1] = (1 - cc) * pc[idx0] + std::sqrt(cc * (2 - cc) *
-        muEffective) * step;
+          muEffective) * step;
 
       if (iterate.n_rows > iterate.n_cols)
       {
         C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
-          (pc[idx1] * pc[idx1].t());
+            (pc[idx1] * pc[idx1].t());
       }
       else
       {
         C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
-          (pc[idx1].t() * pc[idx1]);
+            (pc[idx1].t() * pc[idx1]);
       }
     }
     else
@@ -301,12 +282,12 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
       if (iterate.n_rows > iterate.n_cols)
       {
         C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 * (pc[idx1] *
-          pc[idx1].t() + (cc * (2 - cc)) * C[idx0]);
+            pc[idx1].t() + (cc * (2 - cc)) * C[idx0]);
       }
       else
       {
         C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
-          (pc[idx1].t() * pc[idx1] + (cc * (2 - cc)) * C[idx0]);
+            (pc[idx1].t() * pc[idx1] + (cc * (2 - cc)) * C[idx0]);
       }
     }
 
@@ -314,21 +295,19 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
     {
       for (size_t j = 0; j < mu; ++j)
       {
-        C[idx1] = C[idx1] + cmu * w(j) *
-          pStep[idx(j)] * pStep[idx(j)].t();
+        C[idx1] = C[idx1] + cmu * w(j) * pStep[idx(j)] * pStep[idx(j)].t();
       }
     }
     else
     {
       for (size_t j = 0; j < mu; ++j)
       {
-        C[idx1] = C[idx1] + cmu * w(j) *
-          pStep[idx(j)].t() * pStep[idx(j)];
+        C[idx1] = C[idx1] + cmu * w(j) * pStep[idx(j)].t() * pStep[idx(j)];
       }
     }
 
-    arma::eig_sym(eigval, eigvec, C[idx1]);
-    const arma::uvec negativeEigval = arma::find(eigval < 0, 1);
+    eig_sym(eigval, eigvec, C[idx1]);
+    const UVecType negativeEigval = find(eigval < 0, 1);
     if (!negativeEigval.is_empty())
     {
       if (negativeEigval(0) == 0)
@@ -338,19 +317,19 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
       else
       {
         C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) *
-          arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
-          eigvec.cols(0, negativeEigval(0) - 1).t();
+            diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
+            eigvec.cols(0, negativeEigval(0) - 1).t();
       }
     }
 
     // Output current objective function.
     Info << "CMA-ES: iteration " << i << ", objective " << overallObjective
-      << "." << std::endl;
+        << "." << std::endl;
 
     if (std::isnan(overallObjective) || std::isinf(overallObjective))
     {
       Warn << "CMA-ES: converged to " << overallObjective << "; "
-        << "terminating with failure.  Try a smaller step size?" << std::endl;
+          << "terminating with failure.  Try a smaller step size?" << std::endl;
 
       iterate = transformationPolicy.Transform(iterate);
       Callback::EndOptimization(*this, function, iterate, callbacks...);
@@ -361,7 +340,7 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
     {
       if (steps > patience) {
         Info << "CMA-ES: minimized within tolerance " << tolerance << "; "
-          << "terminating optimization." << std::endl;
+            << "terminating optimization." << std::endl;
 
         iterate = transformationPolicy.Transform(iterate);
         Callback::EndOptimization(*this, function, iterate, callbacks...);

include/ensmallen_bits/cmaes/not_empty_transformation.hpp

@@ -1,42 +0,0 @@
-/**
- * @file not_empty_transformation.hpp
- * @author Suvarsha Chennareddy
- *
- * Check whether TransformationPolicyType is EmptyTransformation.
- *
- * ensmallen is free software; you may redistribute it and/or modify it under
- * the terms of the 3-clause BSD license.  You should have received a copy of
- * the 3-clause BSD license along with ensmallen.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef ENSMALLEN_CMAES_NOT_EMPTY_TRANSFORMATION
-#define ENSMALLEN_CMAES_NOT_EMPTY_TRANSFORMATION
-
-/**
- * This partial specialization is used to throw an exception when the
- * TransformationPolicyType is EmptyTransformation and call a constructor with
- * parameters 'lowerBound' and 'upperBound' otherwise.  This shall be removed
- * when the deprecated constructor is removed in the next major version of
- * ensmallen.
- */
-template<typename T1, typename T2>
-struct NotEmptyTransformation : std::true_type
-{
-  void Assign(T1& obj, double lowerBound, double upperBound)
-  {
-    obj = T1(lowerBound, upperBound);
-  }
-};
-
-template<template<typename...> class T, typename... A, typename... B>
-struct NotEmptyTransformation<T<A...>, T<B...>> : std::false_type
-{
-  void Assign(T<A...>& /* obj */,
-              double /* lowerBound */,
-              double /* upperBound */)
-  {
-    throw std::logic_error("TransformationPolicyType is EmptyTransformation");
-  }
-};
-
-#endif

include/ensmallen_bits/cmaes/pop_cmaes.hpp

@@ -6,7 +6,7 @@
  * Definition of the IPOP Covariance Matrix Adaptation Evolution Strategy
  * as proposed by A. Auger and N. Hansen in "A Restart CMA Evolution
  * Strategy With Increasing Population Size" and BIPOP Covariance Matrix
- * Adaptation Evolution Strategy as proposed by N. Hansen in "Benchmarking 
+ * Adaptation Evolution Strategy as proposed by N. Hansen in "Benchmarking
  * a BI-population CMA-ES on the BBOB-2009 function testbed".
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -24,55 +24,59 @@ namespace ens {
 /**
  * Population-based CMA-ES (POP-CMA-ES) that can operate as either IPOP-CMA-ES
  * or BIPOP-CMA-ES based on a flag.
- * 
+ *
  * IPOP CMA-ES is a variant of the stochastic search algorithm
  * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
- * IPOP CMA-ES, also known as CMAES with increasing population size, 
+ * IPOP CMA-ES, also known as CMAES with increasing population size,
  * incorporates a restart strategy that involves gradually increasing
- * the population size. This approach is specifically designed to 
+ * the population size. This approach is specifically designed to
  * enhance the performance of CMA-ES on multi-modal functions.
  *
  * For more information, please refer to:
  *
  * @code
  * @INPROCEEDINGS{1554902,
- *   author={Auger, A. and Hansen, N.},
- *   booktitle={2005 IEEE Congress on Evolutionary Computation},
- *   title={A restart CMA evolution strategy with increasing population size},
- *   year={2005},
- *   volume={2},
- *   number={},
- *   pages={1769-1776 Vol. 2},
- *   doi={10.1109/CEC.2005.1554902}}
+ *   author    = {Auger, A. and Hansen, N.},
+ *   booktitle = {2005 IEEE Congress on Evolutionary Computation},
+ *   title     = {A restart CMA evolution strategy with increasing population
+ *                size},
+ *   year      = {2005},
+ *   volume    = {2},
+ *   number    = {},
+ *   pages     = {1769-1776 Vol. 2},
+ *   doi       = {10.1109/CEC.2005.1554902}}
  * @endcode
- * 
+ *
  * IPOP CMA-ES can optimize separable functions.  For more details, see the
  * documentation on function types included with this distribution or on the
  * ensmallen website.
- * 
+ *
  * BI-Population CMA-ES is a variant of the stochastic search algorithm
  * CMA-ES - Covariance Matrix Adaptation Evolution Strategy.
- * It implements a dual restart strategy with varying population sizes: one 
+ * It implements a dual restart strategy with varying population sizes: one
  * increasing and one with smaller, varied sizes. This BI-population approach
- * is designed to optimize performance on multi-modal function testbeds by 
+ * is designed to optimize performance on multi-modal function testbeds by
  * leveraging different exploration and exploitation dynamics.
  *
  * For more information, please refer to:
  *
  * @code
  * @inproceedings{hansen2009benchmarking,
- *   title={Benchmarking a BI-population CMA-ES on the BBOB-2009 function testbed},
- *   author={Hansen, Nikolaus},
- *   booktitle={Proceedings of the 11th annual conference companion on genetic and evolutionary computation conference: late breaking papers},
- *   pages={2389--2396},
- *   year={2009}}
+ *   title     = {Benchmarking a BI-population CMA-ES on the BBOB-2009 function
+ *                testbed},
+ *   author    = {Hansen, Nikolaus},
+ *   booktitle = {Proceedings of the 11th annual conference companion on genetic
+ *                and evolutionary computation conference: late breaking
+ *                papers},
+ *   pages     = {2389--2396},
+ *   year      = {2009}}
  * @endcode
  *
  * BI-Population CMA-ES can efficiently handle separable, multimodal, and weak
- * structure functions across various dimensions, as demonstrated in the 
+ * structure functions across various dimensions, as demonstrated in the
  * comprehensive results of the BBOB-2009 function testbed. The optimizer
- * utilizes an interlaced multistart strategy to balance between broad 
- * exploration and intensive exploitation, adjusting population sizes and 
+ * utilizes an interlaced multistart strategy to balance between broad
+ * exploration and intensive exploitation, adjusting population sizes and
  * step-sizes dynamically.
  */
 template<typename SelectionPolicyType = FullSelection,
@@ -83,15 +87,15 @@ class POP_CMAES : public CMAES<SelectionPolicyType, TransformationPolicyType>
  public:
   /**
    * Construct the POP-CMA-ES optimizer with the given parameters.
-   * Other than the same CMA-ES parameters, it also adds the maximum number of 
-   * restarts, the increase in population factor, the maximum number of 
+   * Other than the same CMA-ES parameters, it also adds the maximum number of
+   * restarts, the increase in population factor, the maximum number of
    * evaluations, as well as a flag indicating to use BIPOP or not.
    * The suggested values are not necessarily good for the given problem, so it
    * is suggested that the values used be tailored to the task at hand. The
    * maximum number of iterations refers to the maximum number of points that
    * are processed (i.e., one iteration equals one point; one iteration does not
    * equal one pass over the dataset).
-   * 
+   *
    * @param lambda The initial population size (0 use the default size).
    * @param transformationPolicy Instantiated transformation policy used to
    *    map the coordinates to the desired domain.
@@ -107,7 +111,7 @@ class POP_CMAES : public CMAES<SelectionPolicyType, TransformationPolicyType>
    * @param maxFunctionEvaluations Maximum number of function evaluations.
    */
   POP_CMAES(const size_t lambda = 0,
-            const TransformationPolicyType& transformationPolicy = 
+            const TransformationPolicyType& transformationPolicy =
                  TransformationPolicyType(),
             const size_t batchSize = 32,
             const size_t maxIterations = 1000,
@@ -161,11 +165,13 @@ class POP_CMAES : public CMAES<SelectionPolicyType, TransformationPolicyType>
 // Define IPOP_CMAES and BIPOP_CMAES using the POP_CMAES template
 template<typename SelectionPolicyType = FullSelection,
          typename TransformationPolicyType = EmptyTransformation<>>
-using IPOP_CMAES = POP_CMAES<SelectionPolicyType, TransformationPolicyType, false>;
+using IPOP_CMAES = POP_CMAES<
+    SelectionPolicyType, TransformationPolicyType, false>;
 
 template<typename SelectionPolicyType = FullSelection,
          typename TransformationPolicyType = EmptyTransformation<>>
-using BIPOP_CMAES = POP_CMAES<SelectionPolicyType, TransformationPolicyType, true>;
+using BIPOP_CMAES = POP_CMAES<
+    SelectionPolicyType, TransformationPolicyType, true>;
 
 } // namespace ens
 

include/ensmallen_bits/cmaes/pop_cmaes_impl.hpp

@@ -6,7 +6,7 @@
  * Implementation of the IPOP Covariance Matrix Adaptation Evolution Strategy
  * as proposed by A. Auger and N. Hansen in "A Restart CMA Evolution
  * Strategy With Increasing Population Size" and BIPOP Covariance Matrix
- * Adaptation Evolution Strategy as proposed by N. Hansen in "Benchmarking 
+ * Adaptation Evolution Strategy as proposed by N. Hansen in "Benchmarking
  * a BI-population CMA-ES on the BBOB-2009 function testbed".
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -48,7 +48,7 @@ template<typename SelectionPolicyType,
          typename TransformationPolicyType,
          bool UseBIPOPFlag>
 template<typename SeparableFunctionType, typename MatType, typename... CallbackTypes>
-typename MatType::elem_type POP_CMAES<SelectionPolicyType, 
+typename MatType::elem_type POP_CMAES<SelectionPolicyType,
     TransformationPolicyType, UseBIPOPFlag>::Optimize(
     SeparableFunctionType& function,
     MatType& iterateIn,
@@ -65,9 +65,9 @@ typename MatType::elem_type POP_CMAES<SelectionPolicyType,
 
   // First single run with default population size
   MatType iterate = iterateIn;
-  ElemType overallObjective = CMAES<SelectionPolicyType, 
-      TransformationPolicyType>::Optimize(function, iterate, sbc, 
-                                          callbacks...);
+  ElemType overallObjective = CMAES<SelectionPolicyType,
+      TransformationPolicyType>::Optimize(function, iterate, sbc,
+          callbacks...);
 
   overallSBC = sbc;
   ElemType objective;
@@ -85,7 +85,7 @@ typename MatType::elem_type POP_CMAES<SelectionPolicyType,
 
   while (restart < maxRestarts)
   {
-    if (!UseBIPOPFlag || largePopulationBudget <= smallPopulationBudget || 
+    if (!UseBIPOPFlag || largePopulationBudget <= smallPopulationBudget ||
         restart == 0 || restart == maxRestarts - 1)
     {
       // Large population regime (IPOP or BIPOP)
@@ -95,12 +95,12 @@ typename MatType::elem_type POP_CMAES<SelectionPolicyType,
 
       Info << "POP-CMA-ES: restart " << restart << ", large population size" <<
           " (lambda): " << this->PopulationSize() << "." << std::endl;
-      
+
       iterate = iterateIn;
 
       // Optimize using the CMAES object.
-      objective = CMAES<SelectionPolicyType, 
-          TransformationPolicyType>::Optimize(function, iterate, sbc, 
+      objective = CMAES<SelectionPolicyType,
+          TransformationPolicyType>::Optimize(function, iterate, sbc,
           callbacks...);
 
       evaluations = this->FunctionEvaluations();
@@ -110,10 +110,10 @@ typename MatType::elem_type POP_CMAES<SelectionPolicyType,
     {
       // Small population regime (BIPOP only)
       double u = arma::randu<double>();
-      size_t smallLambda = static_cast<size_t>(defaultLambda * std::pow(0.5 * 
+      size_t smallLambda = static_cast<size_t>(defaultLambda * std::pow(0.5 *
           currentLargeLambda / defaultLambda, u * u));
       double stepSizeSmall = 2 * std::pow(10, -2 * arma::randu<double>());
-      
+
       this->PopulationSize() = smallLambda;
       this->StepSize() = stepSizeSmall;
 
@@ -121,10 +121,10 @@ typename MatType::elem_type POP_CMAES<SelectionPolicyType,
           " size (lambda): " << this->PopulationSize() << "." << std::endl;
 
       iterate = iterateIn;
-      
+
       // Optimize using the CMAES object.
-      objective = CMAES<SelectionPolicyType, 
-          TransformationPolicyType>::Optimize(function, iterate, sbc, 
+      objective = CMAES<SelectionPolicyType,
+          TransformationPolicyType>::Optimize(function, iterate, sbc,
                                               callbacks...);
 
       evaluations = this->FunctionEvaluations();
@@ -160,4 +160,4 @@ typename MatType::elem_type POP_CMAES<SelectionPolicyType,
 
 } // namespace ens
 
-#endif
+#endif

include/ensmallen_bits/cne/cne.hpp

@@ -149,14 +149,14 @@ class CNE
 
  private:
   //! Reproduce candidates to create the next generation.
-  template<typename MatType>
+  template<typename MatType, typename IndexType>
   void Reproduce(std::vector<MatType>& population,
                  const MatType& fitnessValues,
-                 arma::uvec& index);
+                 IndexType& index);
 
   //! Modify weights with some noise for the evolution of next generation.
-  template<typename MatType>
-  void Mutate(std::vector<MatType>& population, arma::uvec& index);
+  template<typename MatType, typename IndexType>
+  void Mutate(std::vector<MatType>& population, IndexType& index);
 
   /**
    * Crossover parents and create new childs. Two parents create two new childs.

include/ensmallen_bits/cne/cne_impl.hpp

@@ -47,6 +47,7 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
   typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename ForwardType<MatType>::uvec UVecType;
 
   // Make sure that we have the methods that we need.  Long name...
   traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType,
@@ -56,7 +57,7 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
   // Vector of fitness values corresponding to each candidate.
   BaseMatType fitnessValues;
   //! Index of sorted fitness values.
-  arma::uvec index;
+  UVecType index;
 
   // Make sure for evolution to work at least four candidates are present.
   if (populationSize < 4)
@@ -93,8 +94,8 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
   std::vector<BaseMatType> population;
   for (size_t i = 0 ; i < populationSize; ++i)
   {
-    population.push_back(arma::randn<BaseMatType>(iterate.n_rows,
-        iterate.n_cols) + iterate);
+    population.push_back(BaseMatType(iterate.n_rows, iterate.n_cols,
+        GetFillType<MatType>::randn) + iterate);
   }
 
   // Store the number of elements in the objective matrix.
@@ -164,13 +165,13 @@ typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
 }
 
 //! Reproduce candidates to create the next generation.
-template<typename MatType>
+template<typename MatType, typename IndexType>
 inline void CNE::Reproduce(std::vector<MatType>& population,
                            const MatType& fitnessValues,
-                           arma::uvec& index)
+                           IndexType& index)
 {
   // Sort fitness values. Smaller fitness value means better performance.
-  index = arma::sort_index(fitnessValues);
+  index = sort_index(fitnessValues);
 
   // First parent.
   size_t mom;
@@ -241,17 +242,20 @@ inline void CNE::Crossover(std::vector<MatType>& population,
 }
 
 //! Modify weights with some noise for the evolution of next generation.
-template<typename MatType>
-inline void CNE::Mutate(std::vector<MatType>& population, arma::uvec& index)
+template<typename MatType, typename IndexType>
+inline void CNE::Mutate(std::vector<MatType>& population, IndexType& index)
 {
+  typedef typename MatType::elem_type ElemType;
+
   // Mutate the whole matrix with the given rate and probability.
   // The best candidate is not altered.
   for (size_t i = 1; i < populationSize; i++)
   {
-    population[index(i)] += (arma::randu<MatType>(population[index(i)].n_rows,
-        population[index(i)].n_cols) < mutationProb) %
-        (mutationSize * arma::randn<MatType>(population[index(i)].n_rows,
-        population[index(i)].n_cols));
+    population[index(i)] += conv_to<MatType>::from(
+        randu<MatType>(population[index(i)].n_rows,
+        population[index(i)].n_cols) < ElemType(mutationProb)) %
+        (ElemType(mutationSize) * MatType(population[index(i)].n_rows,
+        population[index(i)].n_cols, GetFillType<MatType>::randn));
   }
 }
 

include/ensmallen_bits/de/de.hpp

@@ -45,10 +45,10 @@ namespace ens {
  *
  * @code
  * @techreport{storn1995,
- *   title    = {Differential Evolution—a simple and efficient adaptive scheme
- *               for global optimization over continuous spaces},
- *   author   = {Storn, Rainer and Price, Kenneth},
- *   year     = 1995
+ *   title  = {Differential Evolution—a simple and efficient adaptive scheme
+ *             for global optimization over continuous spaces},
+ *   author = {Storn, Rainer and Price, Kenneth},
+ *   year   = 1995
  * }
  * @endcode
  *

include/ensmallen_bits/de/de_impl.hpp

@@ -40,14 +40,16 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
   typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename ForwardType<MatType>::vec ColType;
 
   BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // Population matrix. Each column is a candidate.
   std::vector<BaseMatType> population;
   population.resize(populationSize);
+
   // Vector of fitness values corresponding to each candidate.
-  arma::Col<ElemType> fitnessValues;
+  ColType fitnessValues;
 
   // Make sure that we have the methods that we need.  Long name...
   traits::CheckArbitraryFunctionTypeAPI<
@@ -57,13 +59,13 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
   // Population Size must be at least 3 for DE to work.
   if (populationSize < 3)
   {
-    throw std::logic_error("CNE::Optimize(): population size should be at least"
+    throw std::logic_error("DE::Optimize(): population size should be at least"
         " 3!");
   }
 
   // Initialize helper variables.
   fitnessValues.set_size(populationSize);
-  ElemType lastBestFitness = DBL_MAX;
+  ElemType lastBestFitness = std::numeric_limits<ElemType>::max();
   BaseMatType bestElement;
 
   // Controls early termination of the optimization process.
@@ -82,7 +84,7 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
 
     if (fitnessValues[i] < lastBestFitness)
     {
-      lastBestFitness = fitnessValues[i];
+      lastBestFitness = ElemType(fitnessValues[i]);
       bestElement = population[i];
     }
   }
@@ -111,16 +113,17 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
       while (m == member && m == l);
 
       // Generate new "mutant" from two randomly chosen members.
-      BaseMatType mutant = bestElement + differentialWeight *
+      BaseMatType mutant = bestElement + ElemType(differentialWeight) *
           (population[l] - population[m]);
 
       // Perform crossover.
-      const BaseMatType cr = arma::randu<BaseMatType>(iterate.n_rows);
+      BaseMatType cr;
+      cr.randu(iterate.n_rows, 1);
       for (size_t it = 0; it < iterate.n_rows; it++)
       {
-        if (cr[it] >= crossoverRate)
+        if (cr[it] >= ElemType(crossoverRate))
         {
-          mutant[it] = iterate[it];
+          mutant(it) = ElemType(iterate(it));
         }
       }
 
@@ -158,7 +161,7 @@ typename MatType::elem_type DE::Optimize(FunctionType& function,
     }
 
     // Update helper variables.
-    lastBestFitness = fitnessValues.min();
+    lastBestFitness = ElemType(fitnessValues.min());
     for (size_t it = 0; it < populationSize; it++)
     {
       if (fitnessValues[it] == lastBestFitness)

include/ensmallen_bits/delta_bar_delta/delta_bar_delta.hpp

@@ -0,0 +1,179 @@
+/**
+ * @file delta_bar_delta.hpp
+ * @author Ranjodh Singh
+ *
+ * Class wrapper for the DeltaBarDelta update policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DELTA_BAR_DELTA_HPP
+#define ENSMALLEN_DELTA_BAR_DELTA_HPP
+
+#include <ensmallen_bits/gradient_descent/gradient_descent.hpp>
+#include "update_policies/delta_bar_delta_update.hpp"
+
+namespace ens {
+
+/**
+ * DeltaBarDelta optimizer.
+ *
+ * A heuristic designed to accelerate convergence by
+ * adapting the learning rate of each parameter individually.
+ *
+ * According to the Delta-Bar-Delta update:
+ *
+ * - If the current gradient and the exponential average of
+ *   past gradients corresponding to a parameter have the same
+ *   sign, then the step size for that parameter is incremented by
+ *   \f$\kappa\f$. Otherwise, it is decreased by a proportion \f$\phi\f$
+ *   of its current value (additive increase, multiplicative decrease).
+ *
+ * @note This implementation uses a minStepSize parameter to set a lower
+ *     bound for the learning rate. This prevents the learning rate from
+ *     dropping to zero, which can occur due to floating-point underflow.
+ *     For tasks which require extreme fine-tuning, you may need to lower
+ *     this parameter below its default value (1e-8) in order to allow for
+ *     smaller learning rates.
+ *
+ * @code
+ * @article{jacobs1988increased,
+ *   title     = {Increased Rates of Convergence Through Learning Rate
+ *                Adaptation},
+ *   author    = {Jacobs, Robert A.},
+ *   journal   = {Neural Networks},
+ *   volume    = {1},
+ *   number    = {4},
+ *   pages     = {295--307},
+ *   year      = {1988},
+ *   publisher = {Pergamon}
+ * }
+ * @endcode
+ */
+class DeltaBarDelta
+{
+ public:
+  /**
+   * Construct the DeltaBarDelta optimizer with the given function and
+   * parameters. DeltaBarDelta is very sensitive to its parameters (kappa
+   * and phi) hence a good hyperparameter selection is necessary as its
+   * default may not fit every case.
+   *
+   * @param stepSize Step size (initial).
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param kappa Additive increase constant for step size.
+   * @param phi Multiplicative decrease factor for step size.
+   * @param theta Decay rate for the exponential moving average.
+   * @param minStepSize Minimum allowed step size for any parameter
+   *     (default: 1e-8).
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   */
+  DeltaBarDelta(const double stepSize = 1.0,
+                const size_t maxIterations = 100000,
+                const double tolerance = 1e-5,
+                const double kappa = 0.2,
+                const double phi = 0.2,
+                const double theta = 0.5,
+                const double minStepSize = 1e-8,
+                const bool resetPolicy = true);
+
+  /**
+   * Optimize the given function using DeltaBarDelta.
+   * The given starting point will be modified to store the finishing
+   * point of the algorithm, and the final objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the initial step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the initial step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the additive increase constant for step size.
+  double Kappa() const { return optimizer.UpdatePolicy().Kappa(); }
+  //! Modify the additive increase constant for step size.
+  double& Kappa() { return optimizer.UpdatePolicy().Kappa(); }
+
+  //! Get the multiplicative decrease factor for step size.
+  double Phi() const { return optimizer.UpdatePolicy().Phi(); }
+  //! Modify the multiplicative decrease factor for step size.
+  double& Phi() { return optimizer.UpdatePolicy().Phi(); }
+
+  //! Get the decay rate for the exponential moving average.
+  double Theta() const { return optimizer.UpdatePolicy().Theta(); }
+  //! Modify the decay rate for the exponential moving average.
+  double& Theta() { return optimizer.UpdatePolicy().Theta(); }
+
+  //! Get the minimum allowed step size for any parameter.
+  double MinStepSize() const { return optimizer.UpdatePolicy().MinStepSize(); }
+  //! Modify the minimum allowed step size for any parameter.
+  double& MinStepSize() { return optimizer.UpdatePolicy().MinStepSize(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The GradientDescentType object with DeltaBarDelta policy.
+  GradientDescentType<DeltaBarDeltaUpdate, NoDecay> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "delta_bar_delta_impl.hpp"
+
+#endif // ENSMALLEN_DELTA_BAR_DELTA_HPP

include/ensmallen_bits/delta_bar_delta/delta_bar_delta_impl.hpp

@@ -0,0 +1,41 @@
+/**
+ * @file delta_bar_delta_impl.hpp
+ * @author Ranjodh Singh
+ *
+ * Implementation of DeltaBarDelta class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DELTA_BAR_DELTA_IMPL_HPP
+#define ENSMALLEN_DELTA_BAR_DELTA_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "./delta_bar_delta.hpp"
+
+namespace ens {
+
+inline DeltaBarDelta::DeltaBarDelta(
+    const double stepSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const double kappa,
+    const double phi,
+    const double theta,
+    const double minStepSize,
+    const bool resetPolicy) : 
+    optimizer(stepSize,
+              maxIterations,
+              tolerance,
+              DeltaBarDeltaUpdate(stepSize, kappa, phi, theta, minStepSize),
+              NoDecay(),
+              resetPolicy)
+{
+  /* Nothing to do. */
+}
+
+} // namespace ens
+
+#endif // ENSMALLEN_DELTA_BAR_DELTA_IMPL_HPP

include/ensmallen_bits/delta_bar_delta/momentum_delta_bar_delta.hpp

@@ -0,0 +1,183 @@
+/**
+ * @file momentum_delta_bar_delta.hpp
+ * @author Ranjodh Singh
+ *
+ * Class wrapper for the MomentumDeltaBarDelta update policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_HPP
+#define ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_HPP
+
+#include <ensmallen_bits/gradient_descent/gradient_descent.hpp>
+#include "update_policies/momentum_delta_bar_delta_update.hpp"
+
+namespace ens {
+
+/**
+ * MomentumDeltaBarDelta Optimizer.
+ *
+ * A DeltaBarDelta variant that incorporates the following modifications:
+ *  - In the original DeltaBarDelta, the momentum term (delta_bar) is used
+ *    solely for sign comparison with the current gradient and does not
+ *    participate in the parameter update. In this modified variant, the
+ *    momentum term (velocity) is directly used to update the parameters.
+ *  - Instead of adjusting the step size directly, each parameter maintains
+ *    a gain value initialized to 1.0. Updates apply additive increases or
+ *    multiplicative decreases to this gain. The effective step size for a
+ *    parameter is the product of its initial step size and its current gain.
+ *
+ * Note: This variant originates from optimization of the t-SNE cost function.
+ *
+ * @code
+ * @article{maaten2008visualizing,
+ *   title={Visualizing data using t-SNE},
+ *   author={van der Maaten, Laurens and Hinton, Geoffrey},
+ *   journal={Journal of machine learning research},
+ *   volume={9},
+ *   pages={2579--2605},
+ *   month={11},
+ *   year={2008}
+ * }
+ * @endcode
+ *
+ * @code
+ * @article{jacobs1988increased,
+ *   title     = {Increased Rates of Convergence Through Learning Rate
+ *                Adaptation},
+ *   author    = {Jacobs, Robert A.},
+ *   journal   = {Neural Networks},
+ *   volume    = {1},
+ *   number    = {4},
+ *   pages     = {295--307},
+ *   year      = {1988},
+ *   publisher = {Pergamon}
+ * }
+ * @endcode
+ */
+class MomentumDeltaBarDelta
+{
+ public:
+  /**
+   * Construct the MomentumDeltaBarDelta optimizer with the given function and
+   * parameters.
+   *
+   * @param stepSize Step size (initial).
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param kappa Additive increase constant for step size.
+   * @param phi Multiplicative decrease factor for step size.
+   * @param momentum The momentum decay hyperparameter.
+   * @param minGain Minimum allowed gain (scaling factor) for any parameter
+   *     (default: 1e-8).
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   */
+  MomentumDeltaBarDelta(const double stepSize = 1.0,
+                        const size_t maxIterations = 100000,
+                        const double tolerance = 1e-5,
+                        const double kappa = 0.2,
+                        const double phi = 0.8,
+                        const double momentum = 0.5,
+                        const double minGain = 1e-8,
+                        const bool resetPolicy = true);
+
+  /**
+   * Optimize the given function using MomentumDeltaBarDelta.
+   * The given starting point will be modified to store the finishing
+   * point of the algorithm, and the final objective value is returned.
+   *
+   * @tparam SeparableFunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the initial step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the initial step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the additive increase constant for step size.
+  double Kappa() const { return optimizer.UpdatePolicy().Kappa(); }
+  //! Modify the additive increase constant for step size.
+  double& Kappa() { return optimizer.UpdatePolicy().Kappa(); }
+
+  //! Get the multiplicative decrease factor for step size.
+  double Phi() const { return optimizer.UpdatePolicy().Phi(); }
+  //! Modify the multiplicative decrease factor for step size.
+  double& Phi() { return optimizer.UpdatePolicy().Phi(); }
+
+  //! Get the momentum decay hyperparameter.
+  double Momentum() const { return optimizer.UpdatePolicy().Momentum(); }
+  //! Modify the momentum decay hyperparameter.
+  double& Momentum() { return optimizer.UpdatePolicy().Momentum(); }
+
+  //! Get the minimum allowed gain (scaling factor) for any parameter.
+  double MinGain() const { return optimizer.UpdatePolicy().MinGain(); }
+  //! Modify the minimum allowed gain (scaling factor) for any parameter.
+  double& MinGain() { return optimizer.UpdatePolicy().MinGain(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The GradientDescentType object with MomentumDeltaBarDelta policy.
+  GradientDescentType<MomentumDeltaBarDeltaUpdate, NoDecay> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "momentum_delta_bar_delta_impl.hpp"
+
+#endif // ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_HPP

include/ensmallen_bits/delta_bar_delta/momentum_delta_bar_delta_impl.hpp

@@ -0,0 +1,41 @@
+/**
+ * @file momentum_delta_bar_delta_impl.hpp
+ * @author Ranjodh Singh
+ *
+ * Implementation of MomentumDeltaBarDelta class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP
+#define ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "./momentum_delta_bar_delta.hpp"
+
+namespace ens {
+
+inline MomentumDeltaBarDelta::MomentumDeltaBarDelta(
+    const double stepSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const double kappa,
+    const double phi,
+    const double momentum,
+    const double minGain,
+    const bool resetPolicy) : 
+    optimizer(stepSize,
+              maxIterations,
+              tolerance,
+              MomentumDeltaBarDeltaUpdate(kappa, phi, momentum, minGain),
+              NoDecay(),
+              resetPolicy)
+{
+  /* Nothing to do. */
+}
+
+} // namespace ens
+
+#endif // ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP

include/ensmallen_bits/delta_bar_delta/update_policies/delta_bar_delta_update.hpp

@@ -0,0 +1,197 @@
+/**
+ * @file delta_bar_delta_update.hpp
+ * @author Ranjodh Singh
+ *
+ * DeltaBarDelta update policy for Gradient Descent.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DELTA_BAR_DELTA_UPDATE_HPP
+#define ENSMALLEN_DELTA_BAR_DELTA_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * DeltaBarDelta update policy for Gradient Descent.
+ *
+ * A heuristic designed to accelerate convergence by
+ * adapting the learning rate of each parameter individually.
+ *
+ * According to the Delta-Bar-Delta update:
+ *
+ * - If the current gradient and the exponential average of
+ *   past gradients corresponding to a parameter have the same
+ *   sign, then the step size for that parameter is incremented by
+ *   \f$\kappa\f$. Otherwise, it is decreased by a proportion \f$\phi\f$
+ *   of its current value (additive increase, multiplicative decrease).
+ *
+ * @note This implementation uses a minStepSize parameter to set a lower
+ *     bound for the learning rate. This prevents the learning rate from
+ *     dropping to zero, which can occur due to floating-point underflow.
+ *     For tasks which require extreme fine-tuning, you may need to lower
+ *     this parameter below its default value (1e-8) in order to allow for
+ *     smaller learning rates.
+ *
+ * @code
+ * @article{jacobs1988increased,
+ *   title     = {Increased Rates of Convergence Through Learning Rate
+ *                Adaptation},
+ *   author    = {Jacobs, Robert A.},
+ *   journal   = {Neural Networks},
+ *   volume    = {1},
+ *   number    = {4},
+ *   pages     = {295--307},
+ *   year      = {1988},
+ *   publisher = {Pergamon}
+ * }
+ * @endcode
+ */
+class DeltaBarDeltaUpdate
+{
+ public:
+  /**
+   * Construct the DeltaBarDelta update policy with given parameters.
+   *
+   * @param initialStepSize Initial Step Size.
+   * @param kappa Additive increase constant for step size.
+   * @param phi Multiplicative decrease factor for step size.
+   * @param theta Decay rate for the exponential moving average.
+   * @param minStepSize Minimum allowed step size for any parameter
+   *     (default: 1e-8).
+   */
+  DeltaBarDeltaUpdate(
+      const double initialStepSize,
+      const double kappa,
+      const double phi,
+      const double theta,
+      const double minStepSize = 1e-8) :
+      initialStepSize(initialStepSize),
+      kappa(kappa),
+      phi(phi),
+      theta(theta),
+      minStepSize(minStepSize)
+  {
+    /* Do nothing. */
+  }
+
+  //! Access the initialStepSize hyperparameter.
+  double InitialStepSize() const { return initialStepSize; }
+  //! Modify the initialStepSize hyperparameter.
+  double& InitialStepSize() { return initialStepSize; }
+
+  //! Access the kappa hyperparameter.
+  double Kappa() const { return kappa; }
+  //! Modify the kappa hyperparameter.
+  double& Kappa() { return kappa; }
+
+  //! Access the phi hyperparameter.
+  double Phi() const { return phi; }
+  //! Modify the phi hyperparameter.
+  double& Phi() { return phi; }
+
+  //! Access the theta hyperparameter.
+  double Theta() const { return theta; }
+  //! Modify the theta hyperparameter.
+  double& Theta() { return theta; }
+
+  //! Access the minStepSize hyperparameter.
+  double MinStepSize() const { return minStepSize; }
+  //! Modify the minStepSize hyperparameter.
+  double& MinStepSize() { return minStepSize; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType. This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template <typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    typedef typename MatType::elem_type ElemType;
+
+    /**
+     * This is called by the optimizer method before the start of the iteration
+     * update process.
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(
+        const DeltaBarDeltaUpdate& parent,
+        const size_t rows,
+        const size_t cols) :
+        parent(parent),
+        kappa(ElemType(parent.kappa)),
+        phi(ElemType(parent.phi)),
+        theta(ElemType(parent.theta)),
+        minStepSize(ElemType(parent.minStepSize))
+    {
+      deltaBar.zeros(rows, cols);
+      epsilon.set_size(rows, cols);
+      epsilon.fill(ElemType(parent.InitialStepSize()));
+    }
+
+    /**
+     * Update step for Gradient Descent.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param delta The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double /* stepSize */,
+                const GradType& delta)
+    {
+      const MatType signMatrix = sign(delta % deltaBar);
+
+      epsilon += conv_to<MatType>::from((signMatrix == +1) * kappa -
+          (signMatrix == -1) * phi % epsilon);
+      epsilon.clamp(minStepSize, arma::Datum<ElemType>::inf);
+
+      deltaBar = theta * deltaBar + (1 - theta) * delta;
+      iterate -= epsilon % delta;
+    }
+
+   private:
+    //! The instantiated parent class.
+    const DeltaBarDeltaUpdate& parent;
+
+    //! The exponential average of past gradients.
+    MatType deltaBar;
+    
+    //! Tracks the current step size for each parameter.
+    MatType epsilon;
+
+    // Parent parameters converted to the element type of the matrix.
+    ElemType kappa;
+    ElemType phi;
+    ElemType theta;
+    ElemType minStepSize;
+  };
+
+ private:
+  //! The initialStepSize hyperparameter.
+  double initialStepSize;
+
+  //! The kappa hyperparameter.
+  double kappa;
+
+  //! The phi hyperparameter.
+  double phi;
+
+  //! The theta hyperparameter.
+  double theta;
+
+  //! The minStepSize hyperparameter.
+  double minStepSize;
+};
+
+} // namespace ens
+
+#endif // ENSMALLEN_DELTA_BAR_DELTA_UPDATE_HPP

include/ensmallen_bits/delta_bar_delta/update_policies/momentum_delta_bar_delta_update.hpp

@@ -0,0 +1,178 @@
+/**
+ * @file momentum_delta_bar_delta_update.hpp
+ * @author Ranjodh Singh
+ *
+ * MomentumDeltaBarDelta update policy for Gradient Descent.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_UPDATE_HPP
+#define ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * MomentumDeltaBarDelta update policy for Gradient Descent.
+ *
+ * A DeltaBarDelta variant that incorporates the following modifications:
+ *  - In the original DeltaBarDelta, the momentum term (delta_bar) is used
+ *    solely for sign comparison with the current gradient and does not
+ *    participate in the parameter update. In this modified variant, the
+ *    momentum term (velocity) is directly used to update the parameters.
+ *  - Instead of adjusting the step size directly, each parameter maintains
+ *    a gain value initialized to 1.0. Updates apply additive increases or
+ *    multiplicative decreases to this gain. The effective step size for a
+ *    parameter is the product of its initial step size and its current gain.
+ *
+ * Note: This variant originates from optimization of the t-SNE cost function.
+ *
+ * @code
+ * @article{jacobs1988increased,
+ *   title     = {Increased Rates of Convergence Through Learning Rate
+ *                Adaptation},
+ *   author    = {Jacobs, Robert A.},
+ *   journal   = {Neural Networks},
+ *   volume    = {1},
+ *   number    = {4},
+ *   pages     = {295--307},
+ *   year      = {1988},
+ *   publisher = {Pergamon}
+ * }
+ * @endcode
+ */
+class MomentumDeltaBarDeltaUpdate
+{
+ public:
+  /**
+   * Construct the MomentumDeltaBarDelta update policy with given parameters.
+   *
+   * @param kappa Additive increase constant for step size.
+   * @param phi Multiplicative decrease factor for step size.
+   * @param momentum The momentum decay hyperparameter.
+   * @param minGain Minimum allowed gain (scaling factor) for any parameter
+   *     (default: 1e-8).
+   */
+  MomentumDeltaBarDeltaUpdate(
+      const double kappa = 0.2,
+      const double phi = 0.8,
+      const double momentum = 0.5,
+      const double minGain = 1e-8) :
+      kappa(kappa),
+      phi(phi),
+      momentum(momentum),
+      minGain(minGain)
+  {
+    /* Do nothing. */
+  }
+
+  //! Access the kappa hyperparameter.
+  double Kappa() const { return kappa; }
+  //! Modify the kappa hyperparameter.
+  double& Kappa() { return kappa; }
+
+  //! Access the phi hyperparameter.
+  double Phi() const { return phi; }
+  //! Modify the phi hyperparameter.
+  double& Phi() { return phi; }
+
+  //! Access the momentum hyperparameter.
+  double Momentum() const { return momentum; }
+  //! Modify the momentum hyperparameter.
+  double& Momentum() { return momentum; }
+
+  //! Access the minGain hyperparameter.
+  double MinGain() const { return minGain; }
+  //! Modify the minGain hyperparameter.
+  double& MinGain() { return minGain; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType. This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template <typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    typedef typename MatType::elem_type ElemType;
+
+    /**
+     * This is called by the optimizer method before the start of the iteration
+     * update process.
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(
+        const MomentumDeltaBarDeltaUpdate& parent,
+        const size_t rows,
+        const size_t cols) :
+        parent(parent),
+        kappa(ElemType(parent.kappa)),
+        phi(ElemType(parent.phi)),
+        momentum(ElemType(parent.momentum)),
+        minGain(ElemType(parent.minGain))
+    {
+      gains.ones(rows, cols);
+      velocity.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for Gradient Descent.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      gains += conv_to<MatType>::from(
+          (sign(gradient) != sign(velocity)) * kappa -
+          (sign(gradient) == sign(velocity)) * (1 - phi) % gains);
+      gains.clamp(minGain, arma::Datum<ElemType>::inf);
+
+      velocity = momentum * velocity - (ElemType(stepSize) * gains) % gradient;
+      iterate += velocity;
+    }
+
+   private:
+    //! The instantiated parent class.
+    const MomentumDeltaBarDeltaUpdate& parent;
+
+    //! The gains matrix.
+    MatType gains;
+
+    //! The velocity matrix.
+    MatType velocity;
+
+    // Parent parameters converted to the element type of the matrix.
+    ElemType kappa;
+    ElemType phi;
+    ElemType momentum;
+    ElemType minGain;
+  };
+
+ private:
+  //! The kappa hyperparameter.
+  double kappa;
+
+  //! The phi hyperparameter.
+  double phi;
+
+  //! The momentum hyperparameter.
+  double momentum;
+
+  //! The minGain hyperparameter.
+  double minGain;
+};
+
+} // namespace ens
+
+#endif // ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_UPDATE_HPP

include/ensmallen_bits/demon_adam/demon_adam.hpp

@@ -31,11 +31,11 @@ namespace ens {
  *
  * @code
  * @misc{
- *   title   = {Decaying momentum helps neural network training},
- *   author  = {John Chen and Cameron Wolfe and Zhao Li
- *              and Anastasios Kyrillidis},
- *   url     = {https://arxiv.org/abs/1910.04952}
- *   year    = {2019}
+ *   title  = {Decaying momentum helps neural network training},
+ *   author = {John Chen and Cameron Wolfe and Zhao Li
+ *             and Anastasios Kyrillidis},
+ *   url    = {https://arxiv.org/abs/1910.04952}
+ *   year   = {2019}
  * }
  *
  * DemonAdam can optimize differentiable separable functions. For more details,

include/ensmallen_bits/demon_adam/demon_adam_update.hpp

@@ -90,6 +90,7 @@ class DemonAdamUpdate
     // Convenient typedef.
     typedef typename UpdateRule::template Policy<MatType, GradType>
         InstUpdateRuleType;
+    typedef typename MatType::elem_type ElemType;
 
     /**
      * This constructor is called by the SGD Optimize() method before the start
@@ -103,7 +104,8 @@ class DemonAdamUpdate
            const size_t rows,
            const size_t cols) :
       parent(parent),
-      adamUpdate(new InstUpdateRuleType(parent.adamUpdateInst, rows, cols))
+      adamUpdate(new InstUpdateRuleType(parent.adamUpdateInst, rows, cols)),
+      betaInit(ElemType(parent.betaInit))
     { /* Nothing to do here */ }
 
     /**
@@ -125,12 +127,12 @@ class DemonAdamUpdate
                 const double stepSize,
                 const GradType& gradient)
     {
-      double decayRate = 1;
+      ElemType decayRate = 1;
       if (parent.t > 0)
-        decayRate = 1.0 - (double) parent.t / (double) parent.T;
+        decayRate = 1 - ElemType((double) parent.t / (double) parent.T);
 
-      const double betaDecay = parent.betaInit * decayRate;
-      const double beta = betaDecay / ((1.0 - parent.betaInit) + betaDecay);
+      const ElemType betaDecay = betaInit * decayRate;
+      const ElemType beta = betaDecay / ((1 - betaInit) + betaDecay);
 
       // Perform the update.
       iterate *= beta;
@@ -143,11 +145,14 @@ class DemonAdamUpdate
     }
 
    private:
-    //! Instantiated parent object.
+    // Instantiated parent object.
     DemonAdamUpdate<UpdateRule>& parent;
 
-    //! The update policy.
+    // The update policy.
     InstUpdateRuleType* adamUpdate;
+
+    // Optimizer parameter converted to the element type of the optimization.
+    ElemType betaInit;
   };
 
  private:

include/ensmallen_bits/demon_sgd/demon_sgd.hpp

@@ -25,11 +25,11 @@ namespace ens {
  *
  * @code
  * @misc{
- *   title   = {Decaying momentum helps neural network training},
- *   author  = {John Chen and Cameron Wolfe and Zhao Li
- *              and Anastasios Kyrillidis},
- *   url     = {https://arxiv.org/abs/1910.04952}
- *   year    = {2019}
+ *   title  = {Decaying momentum helps neural network training},
+ *   author = {John Chen and Cameron Wolfe and Zhao Li
+ *             and Anastasios Kyrillidis},
+ *   url    = {https://arxiv.org/abs/1910.04952}
+ *   year   = {2019}
  * }
  *
  * DemonSGD can optimize differentiable separable functions. For more details,

include/ensmallen_bits/demon_sgd/demon_sgd_update.hpp

@@ -78,6 +78,8 @@ class DemonSGDUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -89,7 +91,8 @@ class DemonSGDUpdate
     Policy(DemonSGDUpdate& parent,
            const size_t /* rows */,
            const size_t /* cols */) :
-      parent(parent)
+      parent(parent),
+      betaInit(ElemType(parent.betaInit))
     { /* Nothing to do here */ }
 
     /**
@@ -103,34 +106,37 @@ class DemonSGDUpdate
                 const double stepSize,
                 const GradType& gradient)
     {
-      double decayRate = 1;
+      ElemType decayRate = 1;
       if (parent.t > 0)
-        decayRate = 1.0 - (double) parent.t / (double) parent.T;
+        decayRate = 1 - ElemType((double) parent.t / (double) parent.T);
 
-      const double betaDecay = parent.betaInit * decayRate;
-      const double beta = betaDecay / ((1.0 - parent.betaInit) + betaDecay);
+      const ElemType betaDecay = betaInit * decayRate;
+      const ElemType beta = betaDecay / ((1 - betaInit) + betaDecay);
 
       // Perform the update.
       iterate *= beta;
-      iterate -= stepSize * gradient;
+      iterate -= ElemType(stepSize) * gradient;
 
       // Increment the iteration counter variable.
       ++parent.t;
     }
 
    private:
-    //! Instantiated parent object.
+    // Instantiated parent object.
     DemonSGDUpdate& parent;
+
+    // Optimizer parameter converted to the element type of the optimization.
+    ElemType betaInit;
   };
 
  private:
-  //! The number of momentum iterations.
+  // The number of momentum iterations.
   size_t T;
 
-  //! Initial momentum coefficient.
+  // Initial momentum coefficient.
   double betaInit;
 
-  //! The number of iterations.
+  // The number of iterations.
   size_t t;
 };
 

include/ensmallen_bits/ens_version.hpp

@@ -12,20 +12,20 @@
 
 // This follows the Semantic Versioning pattern defined in https://semver.org/.
 
-#define ENS_VERSION_MAJOR 2
+#define ENS_VERSION_MAJOR 3
 // The minor version is two digits so regular numerical comparisons of versions
 // work right.  The first minor version of a release is always 10.
-#define ENS_VERSION_MINOR 22
-#define ENS_VERSION_PATCH 2
+#define ENS_VERSION_MINOR 11
+#define ENS_VERSION_PATCH 0
 // If this is a release candidate, it will be reflected in the version name
 // (i.e. the version name will be "RC1", "RC2", etc.).  Otherwise the version
 // name will typically be a seemingly arbitrary set of words that does not
 // contain the capitalized string "RC".
-#define ENS_VERSION_NAME "E-Bike Excitement"
+#define ENS_VERSION_NAME "Sunny Day"
 // Incorporate the date the version was released.
 #define ENS_VERSION_YEAR "2025"
-#define ENS_VERSION_MONTH "04"
-#define ENS_VERSION_DAY "30"
+#define ENS_VERSION_MONTH "12"
+#define ENS_VERSION_DAY "15"
 
 namespace ens {
 

include/ensmallen_bits/eve/eve.hpp

@@ -106,7 +106,7 @@ class Eve
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/eve/eve_impl.hpp

@@ -49,8 +49,8 @@ template<typename SeparableFunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 Eve::Optimize(SeparableFunctionType& function,
               MatType& iterateIn,
               CallbackTypes&&... callbacks)
@@ -126,29 +126,37 @@ Eve::Optimize(SeparableFunctionType& function,
     if (terminate)
       break;
 
-    m *= beta1;
-    m += (1 - beta1) * gradient;
+    m *= ElemType(beta1);
+    m += (1 - ElemType(beta1)) * gradient;
 
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
+    v *= ElemType(beta2);
+    v += (1 - ElemType(beta2)) * (gradient % gradient);
 
-    const double biasCorrection1 = 1.0 - std::pow(beta1, (double) (i + 1));
-    const double biasCorrection2 = 1.0 - std::pow(beta2, (double) (i + 1));
+    const ElemType biasCorrection1 =
+        1 - std::pow(ElemType(beta1), ElemType(i + 1));
+    const ElemType biasCorrection2 =
+        1 - std::pow(ElemType(beta2), ElemType(i + 1));
 
     if (i > 0)
     {
       const ElemType d = std::abs(objective - lastObjective) /
-          (std::min(objective, lastObjective) + epsilon);
+          (std::min(objective, lastObjective) + ElemType(epsilon));
 
-      dt *= beta3;
-      dt += (1 - beta3) * std::min(std::max(d, ElemType(1.0 / clip)),
+      dt *= ElemType(beta3);
+      dt += (1 - ElemType(beta3)) * std::min(std::max(d, ElemType(1.0 / clip)),
           ElemType(clip));
     }
 
     lastObjective = objective;
 
-    iterate -= stepSize / dt * (m / biasCorrection1) /
-        (arma::sqrt(v / biasCorrection2) + epsilon);
+    // TODO: remove in ensmallen 4.0.0.
+    #if defined(ENS_OLD_SEPARABLE_STEP_BEHAVIOR)
+    iterate -= ElemType(stepSize) / dt * (m / biasCorrection1) /
+        (sqrt(v / biasCorrection2) + ElemType(epsilon));
+    #else
+    iterate -= (ElemType(stepSize) / (dt * effectiveBatchSize)) *
+        (m / biasCorrection1) / (sqrt(v / biasCorrection2) + ElemType(epsilon));
+    #endif
 
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
 
@@ -186,13 +194,10 @@ Eve::Optimize(SeparableFunctionType& function,
       terminate |= Callback::BeginEpoch(*this, f, iterate, epoch,
           overallObjective, callbacks...);
 
-      // Reset the counter variables if we will continue.
-      if (i != actualMaxIterations)
-      {
-        lastOverallObjective = overallObjective;
-        overallObjective = 0;
-        currentFunction = 0;
-      }
+      // Reset the counter variables.
+      lastOverallObjective = overallObjective;
+      overallObjective = 0;
+      currentFunction = 0;
 
       if (shuffle) // Determine order of visitation.
         f.Shuffle();

include/ensmallen_bits/fasta/fasta.hpp

@@ -0,0 +1,220 @@
+/**
+ * @file fasta.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of FASTA (Fast Adaptive Shrinkage/Thresholding Algorithm).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FASTA_FASTA_HPP
+#define ENSMALLEN_FASTA_FASTA_HPP
+
+#include "../fbs/l1_penalty.hpp"
+#include "../fbs/l1_constraint.hpp"
+
+namespace ens {
+
+/**
+ * FASTA (Fast Adaptive Shrinkage/Thresholding Algorithm) is a proximal
+ * gradient optimization technique for optimizing a function of the form
+ *
+ *   h(x) = f(x) + g(x)
+ *
+ * where f(x) is a differentiable function and g(x) is an arbitrary
+ * non-differentiable function.  In such a situation, standard gradient descent
+ * techniques cannot work because of the non-differentiability of g(x).  To work
+ * around this, FASTA takes a _forward step_ that is just a gradient descent
+ * step on f(x), and then a _backward step_ that is the _proximal operator_
+ * corresponding to g(x).  This continues until convergence.
+ *
+ * This implementation of FASTA allows specification of the backward step (or
+ * proximal operator) via the `BackwardStepType` template parameter.  When using
+ * FBS, the differentiable `FunctionType` given to `Optimize()` should be f(x),
+ * *not* the combined function h(x).  g(x) should be specified by the choice of
+ * `BackwardStepType` (e.g. `L1Penalty` or `L1Maximum`).  The `Optimize()`
+ * function will then return optimized coordinates for h(x), not f(x).
+ *
+ * For more information, see the following paper:
+ *
+ * ```
+ * @article{goldstein2014field,
+ *   title={A field guide to forward-backward splitting with a FASTA
+ *       implementation},
+ *   author={Goldstein, Tom and Studer, Christoph and Baraniuk, Richard},
+ *   journal={arXiv preprint arXiv:1411.3406},
+ *   year={2014}
+ * }
+ * ```
+ */
+template<typename BackwardStepType = L1Penalty>
+class FASTA
+{
+ public:
+  /**
+   * Construct the FASTA optimizer with the given options, using a
+   * default-constructed BackwardStepType.
+   */
+  FASTA(const size_t maxIterations = 10000,
+        const double tolerance = 1e-7,
+        const size_t maxLineSearchSteps = 50,
+        const double stepSizeAdjustment = 2.0,
+        const size_t lineSearchLookback = 10,
+        const bool estimateStepSize = true,
+        const size_t estimateTrials = 10,
+        const double maxStepSize = 0.001);
+
+  /**
+   * Construct the FASTA optimizer with the given options.
+   */
+  FASTA(BackwardStepType backwardStepType,
+        const size_t maxIterations = 10000,
+        const double tolerance = 1e-7,
+        const size_t maxLineSearchSteps = 50,
+        const double stepSizeAdjustment = 2.0,
+        const size_t lineSearchLookback = 10,
+        const bool estimateStepSize = true,
+        const size_t estimateTrials = 10,
+        const double maxStepSize = 0.001);
+
+  /**
+   * Optimize the given function using FASTA.  The given starting
+   * point will be modified to store the finishing point of the algorithm,
+   * the final objective value is returned.
+   *
+   * The FunctionType template class must provide the following functions:
+   *
+   *   double Evaluate(const arma::mat& coordinates);
+   *   void Gradient(const arma::mat& coordinates,
+   *                 arma::mat& gradient);
+   *
+   * @tparam FunctionType Type of function to be optimized.
+   * @tparam MatType Type of objective matrix.
+   * @tparam GradType Type of gradient matrix (default is MatType).
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to be optimized.
+   * @param iterate Input with starting point, and will be modified to save
+   *                the output optimial solution coordinates.
+   * @param callbacks Callback functions.
+   * @return Objective value at the final solution.
+   */
+  template<typename FunctionType, typename MatType, typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the backward step object.
+  const BackwardStepType& BackwardStep() const { return backwardStep; }
+  //! Modify the backward step object.
+  BackwardStepType& BackwardStep() { return backwardStep; }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return maxIterations; }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return maxIterations; }
+
+  //! Get the tolerance on the gradient norm for termination.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance on the gradient norm for termination.
+  double& Tolerance() { return tolerance; }
+
+  //! Get the maximum number of line search steps.
+  size_t MaxLineSearchSteps() const { return maxLineSearchSteps; }
+  //! Modify the maximum number of line search steps.
+  size_t& MaxLineSearchSteps() { return maxLineSearchSteps; }
+
+  //! Get the step size adjustment parameter.
+  double StepSizeAdjustment() const { return stepSizeAdjustment; }
+  //! Modify the step size adjustment parameter.
+  double& StepSizeAdjustment() { return stepSizeAdjustment; }
+
+  //! Get the maximum number of iterations to look back during a line search.
+  size_t LineSearchLookback() const { return lineSearchLookback; }
+  //! Modify the maximum number of iterations to look back during a line search.
+  size_t& LineSearchLookback() { return lineSearchLookback; }
+
+  //! Get whether or not to estimate the initial step size.
+  bool EstimateStepSize() const { return estimateStepSize; }
+  //! Modify whether or not to estimate the initial step size.
+  bool& EstimateStepSize() { return estimateStepSize; }
+
+  //! Get the number of trials to use for Lipschitz constant estimation.
+  size_t EstimateTrials() const { return estimateTrials; }
+  //! Modify the number of trials to use for Lipschitz constant estimation.
+  size_t& EstimateTrials() { return estimateTrials; }
+
+  //! Get the maximum step size.  If Optimize() has been called, this will
+  //! contain the estimated maximum step size value.
+  double MaxStepSize() const { return maxStepSize; }
+  //! Modify the step size (ignored if EstimateStepSize() is true).
+  double& MaxStepSize() { return maxStepSize; }
+
+ private:
+  //! Utility function: fill with random values.
+  template<typename MatType>
+  static void RandomFill(MatType& x,
+                         const size_t rows,
+                         const size_t cols,
+                         const typename MatType::elem_type maxVal);
+
+  template<typename eT>
+  static void RandomFill(arma::SpMat<eT>& x,
+                         const size_t rows,
+                         const size_t cols,
+                         const eT maxVal);
+
+  template<typename FunctionType, typename MatType>
+  void EstimateLipschitzStepSize(FunctionType& f, const MatType& x);
+
+  //! The instantiated backward step object.
+  BackwardStepType backwardStep;
+
+  //! The maximum number of allowed iterations.
+  size_t maxIterations;
+
+  //! The tolerance for termination.
+  double tolerance;
+
+  //! The maximum number of line search trials.
+  size_t maxLineSearchSteps;
+
+  //! The step size adjustment parameter for the line search.
+  double stepSizeAdjustment;
+
+  //! The maximum number of iterations to look back during a line search.
+  size_t lineSearchLookback;
+
+  //! Whether or not to try and estimate the initial step size.
+  bool estimateStepSize;
+
+  //! Number of trials to use for initial step size estimation.
+  size_t estimateTrials;
+
+  //! The maximum step size to use (estimated if estimateStepSize is true).
+  double maxStepSize;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "fasta_impl.hpp"
+
+#endif

include/ensmallen_bits/fasta/fasta_impl.hpp

@@ -0,0 +1,549 @@
+/**
+ * @file fasta_impl.hpp
+ * @author Ryan Curtin
+ *
+ * Implementation of FASTA (Fast Adaptive Shrinkage/Thresholding Algorithm).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FASTA_FASTA_IMPL_HPP
+#define ENSMALLEN_FASTA_FASTA_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "fasta.hpp"
+
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+//! Constructor of the FBS class.
+template<typename BackwardStepType>
+FASTA<BackwardStepType>::FASTA(const size_t maxIterations,
+                               const double tolerance,
+                               const size_t maxLineSearchSteps,
+                               const double stepSizeAdjustment,
+                               const size_t lineSearchLookback,
+                               const bool estimateStepSize,
+                               const size_t estimateTrials,
+                               const double maxStepSize) :
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    maxLineSearchSteps(maxLineSearchSteps),
+    stepSizeAdjustment(stepSizeAdjustment),
+    lineSearchLookback(lineSearchLookback),
+    estimateStepSize(estimateStepSize),
+    estimateTrials(estimateTrials),
+    maxStepSize(maxStepSize)
+{
+  // Check estimateSteps parameter.
+  if (estimateStepSize && estimateTrials == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): estimateTrials must be greater"
+        " than 0!");
+  }
+
+  if (lineSearchLookback == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): lineSearchLookback cannot be "
+        "0!");
+  }
+}
+
+template<typename BackwardStepType>
+FASTA<BackwardStepType>::FASTA(BackwardStepType backwardStep,
+                               const size_t maxIterations,
+                               const double tolerance,
+                               const size_t maxLineSearchSteps,
+                               const double stepSizeAdjustment,
+                               const size_t lineSearchLookback,
+                               const bool estimateStepSize,
+                               const size_t estimateTrials,
+                               const double maxStepSize) :
+    backwardStep(std::move(backwardStep)),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    maxLineSearchSteps(maxLineSearchSteps),
+    stepSizeAdjustment(stepSizeAdjustment),
+    lineSearchLookback(lineSearchLookback),
+    estimateStepSize(estimateStepSize),
+    estimateTrials(estimateTrials),
+    maxStepSize(maxStepSize)
+{
+  // Check estimateSteps parameter.
+  if (estimateStepSize && estimateTrials == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): estimateTrials must be greater"
+        " than 0!");
+  }
+
+  if (lineSearchLookback == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): lineSearchLookback cannot be "
+        "0!");
+  }
+}
+
+//! Optimize the function (minimize).
+template<typename BackwardStepType>
+template<typename FunctionType, typename MatType, typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
+FASTA<BackwardStepType>::Optimize(FunctionType& function,
+                                  MatType& iterateIn,
+                                  CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
+  FullFunctionType& f = static_cast<FullFunctionType&>(function);
+
+  // Make sure we have all necessary functions.
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  // Sanity check: make sure lineSearchLookback is valid.
+  if (lineSearchLookback == 0)
+  {
+    throw std::invalid_argument("FASTA::FASTA(): lineSearchLookback cannot be "
+        "0!");
+  }
+
+  // Here we make a copy because we will use std::move() internally, and if
+  // iterateIn is an alias, this is unsafe.  We will copy the final result back
+  // to iterateIn at the end.
+  BaseMatType x(iterateIn);
+
+  // To keep track of the function value.
+  ElemType currentFObj = f.Evaluate(x);
+  ElemType currentGObj = backwardStep.Evaluate(x);
+  ElemType currentObj = currentFObj + currentGObj;
+
+  // This will be the denominator of the normalized residual termination
+  // condition.
+  ElemType firstResidual = ElemType(0);
+
+  // This will be used in the non-monotone line search, to track the last
+  // several function values.
+  arma::Col<ElemType> lastFObjs(lineSearchLookback);
+  lastFObjs.fill(std::numeric_limits<ElemType>::min());
+  size_t currentObjPos = 0;
+
+  BaseGradType g(x.n_rows, x.n_cols);
+  BaseMatType lastXHat; // Used for residual checks.
+  BaseMatType lastX; // Used for residual and alpha reset checks.
+  BaseMatType xHat; // Used for residual checks.
+  BaseMatType lpaX = x; // Used for alpha reset check.
+  ElemType alpha = ElemType(1); // Initialize alpha^1 = 1.
+  ElemType lastAlpha = alpha;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // First, estimate the Lipschitz constant to set the initial/maximum step
+  // size, if the user asked us to.
+  if (estimateStepSize)
+    EstimateLipschitzStepSize(f, x);
+
+  // Keep track of the last step size we used.
+  ElemType currentStepSize = (ElemType) maxStepSize;
+  ElemType lastStepSize = (ElemType) maxStepSize;
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
+  Callback::BeginOptimization(*this, f, x, callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
+  {
+    // During this optimization, we want to optimize h(x) = f(x) + g(x).
+    // f(x) is `f`, but g(x) is specified by `BackwardStepType`.
+
+    // The first step is to compute a step size via a non-monotone line search.
+    // To do this, we need to compute the gradient f'(y) as required by the line
+    // search condition in Eq. (38).  Note that our code does a little sleight
+    // of hand, and so `x` stores what the paper calls `y^k` here.  (See the
+    // code for the adaptive step below.)
+    currentFObj = f.EvaluateWithGradient(x, g);
+    terminate |= Callback::EvaluateWithGradient(*this, f, x, currentFObj, g,
+        callbacks...);
+
+    // Use backtracking non-monotone line search to find the best step size.
+    // This is the version from the FASTA paper, but with a minor modification:
+    // we start our search at the last step size, and allow the search to
+    // increase the step size up to the maximum step size if it can.  This is a
+    // more effective heuristic than simply starting at the largest allowable
+    // step size and shrinking from there, especially in regions where the
+    // gradient norm is small.  It is also more effective than simply starting
+    // at the last step size and shrinking from there, as it prevents getting
+    // "stuck" with a very small step size.
+    bool lsDone = false;
+    size_t lsTrial = 0;
+    bool increasing = false; // Will be set during the first iteration.
+    ElemType lastFObj = ElemType(0);
+    BaseMatType lsLastX; // Only used in increasing mode.
+    BaseMatType lsLastXHat; // Only used in increasing mode.
+    BaseMatType xDiff;
+
+    lastX = std::move(x);
+    lastStepSize = currentStepSize;
+    currentStepSize = std::min(currentStepSize, (ElemType) maxStepSize);
+
+    // Ensure that the last `lineSearchLookback` objective values are recorded
+    // properly.
+    lastFObjs[currentObjPos] = currentFObj;
+    currentObjPos = (currentObjPos + 1) % lineSearchLookback;
+    const ElemType strictMaxFObj = currentFObj;
+    const ElemType maxFObj = lastFObjs.max();
+
+    while (!lsDone && !terminate)
+    {
+      if (lsTrial == maxLineSearchSteps)
+      {
+        if (increasing)
+        {
+          Warn << "FASTA::Optimize(): line search reached maximum number of "
+              << "steps (" << maxLineSearchSteps << "); using step size "
+              << currentStepSize << "." << std::endl;
+          break; // The step size is still valid.
+        }
+        else
+        {
+          Warn << "FASTA::Optimize(): could not find valid step size in range "
+              << "(0, " << maxStepSize << "]!  Terminating optimization."
+              << std::endl;
+          terminate = true;
+          break;
+        }
+      }
+
+      // If the step size has converged to zero, we are done.
+      if (currentStepSize == ElemType(0))
+      {
+        Warn << "FASTA::Optimize(): computed zero step size; terminating "
+            << "optimization." << std::endl;
+        terminate = true;
+        break;
+      }
+
+      // Perform forward update into x.
+      xHat = lastX - currentStepSize * g;
+      // (We must store xHat separately for the residual, so this copy is
+      // necessary.)
+      x = xHat;
+      backwardStep.ProximalStep(x, currentStepSize);
+
+      // Compute objective of new point.
+      const ElemType fObj = f.Evaluate(x);
+      terminate |= Callback::Evaluate(*this, f, x, fObj, callbacks...);
+
+      // Compute the quadratic approximation of the objective (the condition in
+      // Eq. (38)).
+      xDiff = (x - lastX);
+
+      // Note: since we allow the step size to increase, we have to modify the
+      // non-monotone line search a little bit to keep things from diverging.
+      // Specifically, if we are increasing the step size, then we force a
+      // monotone line search (by looking only at the previous function value).
+      // It is only when we are decreasing the step size that we allow
+      // relaxation.
+      const ElemType relaxedCond = maxFObj + dot(xDiff, g) +
+          (1 / (2 * currentStepSize)) * dot(xDiff, xDiff);
+      const ElemType strictCond = strictMaxFObj + dot(xDiff, g) +
+          (1 / (2 * currentStepSize)) * dot(xDiff, xDiff);
+
+      // If we're on the first iteration, we don't know if we should be
+      // searching for a step size by increasing or decreasing the step size.
+      // (Remember that our valid ranges of step sizes are [0, maxStepSize], and
+      // we are starting at lastStepSize.)
+      //
+      // Thus, if the condition is satisfied, let's try increasing the step size
+      // until it's no longer satisfied.  Otherwise, we will have to decrease
+      // the step size.
+      if (lsTrial == 0)
+      {
+        increasing = ((fObj <= strictCond) && (std::isfinite(fObj)));
+      }
+
+      if (increasing)
+      {
+        // If we are in "increasing" mode, then termination occurs on the first
+        // iteration when the strict condition is *not* satisfied (and we use
+        // the last step size).
+        if ((fObj > strictCond) || (!std::isfinite(fObj)))
+        {
+          lsDone = true;
+          x = std::move(lsLastX);
+          xHat = std::move(lsLastXHat);
+          currentFObj = lastFObj;
+          currentStepSize = lastStepSize; // Take one step backwards.
+        }
+        else if (currentStepSize == (ElemType) maxStepSize)
+        {
+          // The condition is still satisfied, but the step size will be too big
+          // if we take another step.  Go back to the maximum step size.
+          lsDone = true;
+          currentFObj = fObj;
+        }
+        else
+        {
+          // The condition is still satisfied; increase the step size.
+          lastStepSize = currentStepSize;
+          currentStepSize *= ElemType(stepSizeAdjustment);
+          lsLastX = std::move(x);
+          lsLastXHat = std::move(xHat);
+          lastFObj = fObj;
+          ++lsTrial;
+        }
+      }
+      else
+      {
+        // If we are in "decreasing" mode, then termination occurs on the first
+        // iteration when the relaxed condition is satisfied.
+        if ((fObj <= relaxedCond) && (std::isfinite(fObj)))
+        {
+          lsDone = true;
+          currentFObj = fObj;
+        }
+        else
+        {
+          // The condition is not yet satisfied; decrease the step size.
+          currentStepSize /= ElemType(stepSizeAdjustment);
+          ++lsTrial;
+        }
+      }
+    }
+
+    if (!lsDone)
+    {
+      // The line search failed, so terminate.
+      Warn << "FASTA::Optimize(): non-monotone line search failed after "
+          << maxLineSearchSteps << " steps; terminating optimization."
+          << std::endl;
+      x = std::move(lastX);
+      terminate = true;
+    }
+
+    // If we terminated during the line search, we are done.
+    if (terminate)
+      break;
+
+    // Now that we have taken a step, compute the full objective by computing
+    // g(x).
+    currentGObj = backwardStep.Evaluate(x);
+    currentObj = currentFObj + currentGObj;
+
+    // Output current objective function.
+    Info << "FASTA::Optimize(): iteration " << i << ", combined objective "
+        << currentObj << " (f(x) = " << currentFObj << ", g(x) = "
+        << currentGObj << "), step size " << currentStepSize << "."
+        << std::endl;
+
+    // Sanity check for divergence.
+    if ((i > 1) && !std::isfinite(currentObj))
+    {
+      Warn << "FASTA::Optimize(): objective diverged to "
+          << currentObj << "; terminating optimization." << std::endl;
+      terminate = true;
+      break;
+    }
+
+    // Now, check for convergence.  The FASTA convergence check uses both the
+    // normalized residual and the relative residual, stopping when either
+    // becomes sufficiently small.  The check depends on x before and after the
+    // proximal step.
+
+    // Compute residual.  This is Eq. (40) in the paper.
+    const ElemType residual = norm(g + (xHat - x) / currentStepSize, 2);
+
+    // If this is the first iteration, store the residual as the first residual.
+    if (i == 1)
+      firstResidual = residual;
+
+    // First, check the normalized residual for convergence.  This is Eq. (43)
+    // in the paper.
+    const ElemType eps = 20 * std::numeric_limits<ElemType>::epsilon();
+    const ElemType normalizedResidual = residual / (firstResidual + eps);
+
+    if ((i < 10) && (normalizedResidual < ElemType(1e-5)))
+    {
+      // Heuristic: sometimes the optimization starts in such an awful place
+      // that we are able to make huge amounts of progress in the first few
+      // iterations.  In this case, reset the firstResidual to the slightly
+      // better point we get to by the tenth iterate.
+      firstResidual = residual;
+    }
+    else if ((i > 10) && (normalizedResidual < tolerance))
+    {
+      Info << "FASTA::Optimize(): normalized residual minimized within "
+          << "tolerance " << tolerance << "; terminating optimization."
+          << std::endl;
+      break;
+    }
+
+    // Next, check the relative residual for convergence.  This is Eq. (42) in
+    // the paper.
+    const ElemType gNorm = norm(g, 2);
+    const ElemType proxStepNorm = norm((xHat - x) / currentStepSize, 2);
+
+    const ElemType relativeResidual = residual /
+        (std::max(gNorm, proxStepNorm) + 20 * eps);
+
+    if (relativeResidual < tolerance)
+    {
+      Info << "FASTA::Optimize(): relative residual minimized within "
+          << "tolerance " << tolerance << "; terminating optimization."
+          << std::endl;
+      break;
+    }
+
+    // Compute updated prediction parameter alpha.
+    lastAlpha = alpha;
+    alpha = (1 + std::sqrt(1 + 4 * std::pow(alpha, ElemType(2)))) / 2;
+
+    // Take a predictive step.
+    BaseMatType y = x + ((lastAlpha - 1) / alpha) * (x - lpaX);
+
+    // Sometimes alpha can get to be too large; this restart scheme is taken
+    // originally from O'Donoghue and Candes, "Adaptive restart for accelerated
+    // gradient schemes", 2012.
+    //
+    // The notation is confusing here when compared with Eq. (37) in the paper.
+    // This is because the paper is poorly notated, although it's not clear much
+    // has been done here to improve things.  To translate:
+    //
+    //    Paper   Code    Explanation
+    //
+    //    y^k     lastX   This is the result of the predictive step on the
+    //                    previous iteration.  In our code, we apply the
+    //                    predictive step to x, which next iteration becomes
+    //                    lastX.
+    //
+    //    x^k     x       This is the iterate before the predictive step, this
+    //                    iteration.
+    //
+    //    x^k-1   lpaX    "Last Pre-Accelerated X"---we have to take a specific
+    //                    step to store this.
+    //
+    const ElemType restartCheck = dot(lastX - x, x - lpaX);
+    if (restartCheck > 0)
+    {
+      Info << "FASTA::Optimize(): alpha too large (" << alpha << "); reset to "
+          << "1." << std::endl;
+      alpha = ElemType(1);
+      lastAlpha = ElemType(1);
+    }
+
+    lpaX = std::move(x);
+    x = std::move(y);
+
+    terminate |= Callback::StepTaken(*this, f, x, callbacks...);
+  }
+
+  if (!terminate)
+  {
+    Info << "FASTA::Optimize(): maximum iterations (" << maxIterations
+        << ") reached; terminating optimization." << std::endl;
+  }
+
+  Callback::EndOptimization(*this, f, x, callbacks...);
+
+  ((BaseMatType&) iterateIn) = x;
+  return currentObj;
+} // Optimize()
+
+template<typename BackwardStepType>
+template<typename MatType>
+void FASTA<BackwardStepType>::RandomFill(
+    MatType& x,
+    const size_t rows,
+    const size_t cols,
+    const typename MatType::elem_type maxVal)
+{
+  x.randu(rows, cols);
+  x *= maxVal;
+}
+
+template<typename BackwardStepType>
+template<typename eT>
+void FASTA<BackwardStepType>::RandomFill(
+    arma::SpMat<eT>& x,
+    const size_t rows,
+    const size_t cols,
+    const eT maxVal)
+{
+  eT density = eT(0.1);
+  // Try and keep the matrix from having too many elements.
+  if (rows * cols > 100000)
+    density = eT(0.01);
+  else if (rows * cols > 1000000)
+    density = eT(0.001);
+  else if (rows * cols > 10000000)
+    density = eT(0.0001);
+
+  x.sprandu(rows, cols, density);
+
+  // Make sure we got at least some nonzero elements...
+  while (x.n_nonzero == 0)
+  {
+    if (x.n_elem < 10)
+      x.sprandu(rows, cols, 1.0);
+    else
+      x.sprandu(rows, cols, 0.5);
+  }
+
+  x *= maxVal;
+}
+
+template<typename BackwardStepType>
+template<typename FunctionType, typename MatType>
+void FASTA<BackwardStepType>::EstimateLipschitzStepSize(
+    FunctionType& f,
+    const MatType& x)
+{
+  typedef typename MatType::elem_type ElemType;
+
+  // Sanity check for estimateSteps parameter.
+  if (estimateTrials == 0)
+  {
+    throw std::invalid_argument("FASTA::Optimize(): estimateTrials must be "
+        "greater than 0!");
+  }
+
+  const ElemType xMax = std::max(ElemType(1), 2 * x.max());
+  ElemType sum = ElemType(0);
+  MatType x1, x2, gx1, gx2;
+
+  for (size_t t = 0; t < estimateTrials; ++t)
+  {
+    RandomFill(x1, x.n_rows, x.n_cols, xMax);
+    RandomFill(x2, x.n_rows, x.n_cols, xMax);
+
+    f.Gradient(x1, gx1);
+    f.Gradient(x2, gx2);
+
+    // Compute a Lipschitz constant estimate.
+    const ElemType lEst = norm(gx1 - gx2, 2) / norm(x1 - x2, 2);
+    sum += lEst;
+  }
+
+  sum /= estimateTrials;
+  if (sum == 0)
+    maxStepSize = std::numeric_limits<ElemType>::max();
+  else
+    maxStepSize = (10 / sum);
+
+  Info << "FASTA::Optimize(): estimated a maximum step size of "
+      << maxStepSize << "." << std::endl;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/fbs/fbs.hpp

@@ -0,0 +1,153 @@
+/**
+ * @file fbs.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of Forward-Backward Splitting (FBS).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_FBS_HPP
+#define ENSMALLEN_FBS_FBS_HPP
+
+#include "l1_penalty.hpp"
+#include "l1_constraint.hpp"
+
+namespace ens {
+
+/**
+ * Forward-Backward Splitting is a proximal gradient optimization technique for
+ * optimizing a function of the form
+ *
+ *   h(x) = f(x) + g(x)
+ *
+ * where f(x) is a differentiable function and g(x) is an arbitrary
+ * non-differentiable function.  In such a situation, standard gradient descent
+ * techniques cannot work because of the non-differentiability of g(x).  To work
+ * around this, FBS takes a _forward step_ that is just a gradient descent step
+ * on f(x), and then a _backward step_ that is the _proximal operator_
+ * corresponding to g(x).  This continues until convergence.
+ *
+ * This implementation of FBS allows specification of the backward step (or
+ * proximal operator) via the `BackwardStepType` template parameter.  When using
+ * FBS, the differentiable `FunctionType` given to `Optimize()` should be f(x),
+ * *not* the combined function h(x).  g(x) should be specified by the choice of
+ * `BackwardStepType` (e.g. `L1Penalty` or `L1Maximum`).  The `Optimize()`
+ * function will then return optimized coordinates for h(x), not f(x).
+ *
+ * For more information, see the following paper:
+ *
+ * ```
+ * @article{goldstein2014field,
+ *   title={A field guide to forward-backward splitting with a FASTA
+ *       implementation},
+ *   author={Goldstein, Tom and Studer, Christoph and Baraniuk, Richard},
+ *   journal={arXiv preprint arXiv:1411.3406},
+ *   year={2014}
+ * }
+ * ```
+ */
+template<typename BackwardStepType = L1Penalty>
+class FBS
+{
+ public:
+  /**
+   * Construct the FBS optimizer with the given options, using a
+   * default-constructed BackwardStepType.
+   */
+  FBS(const double stepSize = 0.001,
+      const size_t maxIterations = 10000,
+      const double tolerance = 1e-10);
+
+  /**
+   * Construct the FBS optimizer with the given options.
+   */
+  FBS(BackwardStepType backwardStepType,
+      const double stepSize = 0.001,
+      const size_t maxIterations = 10000,
+      const double tolerance = 1e-10);
+
+  /**
+   * Optimize the given function using FBS.  The given starting
+   * point will be modified to store the finishing point of the algorithm,
+   * the final objective value is returned.
+   *
+   * FunctionType template class must provide the following functions:
+   *
+   *   double Evaluate(const arma::mat& coordinates);
+   *   void Gradient(const arma::mat& coordinates,
+   *                 arma::mat& gradient);
+   *
+   * @tparam FunctionType Type of function to be optimized.
+   * @tparam MatType Type of objective matrix.
+   * @tparam GradType Type of gradient matrix (default is MatType).
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to be optimized.
+   * @param iterate Input with starting point, and will be modified to save
+   *                the output optimial solution coordinates.
+   * @param callbacks Callback functions.
+   * @return Objective value at the final solution.
+   */
+  template<typename FunctionType, typename MatType, typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the backward step object.
+  const BackwardStepType& BackwardStep() const { return backwardStep; }
+  //! Modify the backward step object.
+  BackwardStepType& BackwardStep() { return backwardStep; }
+
+  //! Get the step size.
+  double StepSize() const { return stepSize; }
+  //! Modify the step size.
+  double& StepSize() { return stepSize; }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return maxIterations; }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return maxIterations; }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return tolerance; }
+
+ private:
+  //! The instantiated backward step object.
+  BackwardStepType backwardStep;
+
+  //! The step size for FBS steps.
+  double stepSize;
+
+  //! The maximum number of allowed iterations.
+  size_t maxIterations;
+
+  //! The tolerance for termination.
+  double tolerance;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "fbs_impl.hpp"
+
+#endif

include/ensmallen_bits/fbs/fbs_impl.hpp

@@ -0,0 +1,144 @@
+/**
+ * @file fbs_impl.hpp
+ * @author Ryan Curtin
+ *
+ * Implementation of Forward-Backward Splitting (FBS).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_FBS_IMPL_HPP
+#define ENSMALLEN_FBS_FBS_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "fbs.hpp"
+
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+//! Constructor of the FBS class.
+template<typename BackwardStepType>
+FBS<BackwardStepType>::FBS(const double stepSize,
+                           const size_t maxIterations,
+                           const double tolerance) :
+    stepSize(stepSize),
+    maxIterations(maxIterations),
+    tolerance(tolerance)
+{ /* Nothing to do. */ }
+
+template<typename BackwardStepType>
+FBS<BackwardStepType>::FBS(BackwardStepType backwardStep,
+                           const double stepSize,
+                           const size_t maxIterations,
+                           const double tolerance) :
+    backwardStep(std::move(backwardStep)),
+    stepSize(stepSize),
+    maxIterations(maxIterations),
+    tolerance(tolerance)
+{ /* Nothing to do. */ }
+
+//! Optimize the function (minimize).
+template<typename BackwardStepType>
+template<typename FunctionType, typename MatType, typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
+FBS<BackwardStepType>::Optimize(FunctionType& function,
+                                MatType& iterateIn,
+                                CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
+  FullFunctionType& f = static_cast<FullFunctionType&>(function);
+
+  // Make sure we have all necessary functions.
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // To keep track of the function value.
+  ElemType currentObjective = std::numeric_limits<ElemType>::max();
+  ElemType currentFObjective = currentObjective;
+  ElemType currentGObjective = currentObjective;
+  ElemType lastObjective = currentObjective;
+
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
+  Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
+  {
+    // During this optimization, we want to optimize h(x) = f(x) + g(x).
+    // f(x) is `f`, but g(x) is specified by `BackwardStepType`.
+
+    // First compute f(x) and f'(x).
+    currentFObjective = f.EvaluateWithGradient(iterate, gradient);
+    // Now compute g(x) to get the full objective.
+    currentGObjective = backwardStep.Evaluate(iterate);
+
+    lastObjective = currentObjective;
+    currentObjective = currentFObjective + currentGObjective;
+
+    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        currentObjective, gradient, callbacks...);
+
+    // Output current objective function.
+    Info << "FBS::Optimize(): iteration " << i << ", combined objective "
+        << currentObjective << " (f(x) = " << currentFObjective << ", g(x) = "
+        << currentGObjective << ")." << std::endl;
+
+    // Check for convergence.
+    if ((i > 1) && (std::abs(currentObjective - lastObjective) < tolerance))
+    {
+      Info << "FBS::Optimize(): minimized within objective tolerance "
+          << tolerance << "; terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
+      return currentObjective;
+    }
+
+    if ((i > 1) && !std::isfinite(currentObjective))
+    {
+      Warn << "FBS::Optimize(): objective diverged to " << currentObjective
+          << "; terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
+      return currentObjective;
+    }
+
+    // Perform forward update.
+    iterate -= ElemType(stepSize) * gradient;
+    // Now perform backward step (proximal update).
+    backwardStep.ProximalStep(iterate, stepSize);
+
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
+  }
+
+  if (!terminate)
+  {
+    Info << "FBS::Optimize(): maximum iterations (" << maxIterations
+        << ") reached; terminating optimization." << std::endl;
+  }
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
+  return currentObjective;
+} // Optimize()
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/fbs/l1_constraint.hpp

@@ -0,0 +1,81 @@
+/**
+ * @file l1_constraint.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of the proximal operator for the L1 constraint.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_L1_CONSTRAINT_HPP
+#define ENSMALLEN_FBS_L1_CONSTRAINT_HPP
+
+namespace ens {
+
+/**
+ * The L1Constraint applies a specific constraint that the L1 norm of the
+ * parameters must be less than or equal to the given lambda value.
+ *
+ * Implementationally, this means that the proximal step is a projection onto
+ * the L1 ball of radius lambda.  If the constraint is satisfied, `Evaluate()`
+ * will return 0.  Otherwise, it will return infinity.
+ *
+ * This class is meant to be used with the FBS optimizer, and any other
+ * optimizer that uses a proximal operator/step.
+ */
+class L1Constraint
+{
+ public:
+  /**
+   * Construct an L1Constraint with the given maximum L1 norm for the
+   * coordinates (lambda).
+   */
+  L1Constraint(const double lambda = 0.0);
+
+  /**
+   * If the L1 norm of the coordinates is less than or equal to lambda, this
+   * returns 0.  Otherwise, it returns infinity.
+   */
+  template<typename MatType>
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const;
+
+  /**
+   * Apply a proximal step to the given `coordinates`, assuming that the forward
+   * step took a step of size `stepSize`.  This projects `coordinates` back onto
+   * the surface of the L1-ball with radius `lambda`, if the L1 norm of
+   * `coordinates` is greater than `lambda`.
+   *
+   * This may apply the proximal step multiple times to account for numerical
+   * stability issues during projection.
+   */
+  template<typename MatType>
+  void ProximalStep(MatType& coordinates, const double stepSize) const;
+
+  //! Get the L1 constraint to use when applying the proximal step.
+  double Lambda() const { return lambda; }
+  //! Modify the L1 constraint to use when applying the proximal step.
+  double& Lambda() { return lambda; }
+
+ private:
+  //! The L1 constraint value to use.
+  double lambda;
+
+  //! Helper function: extract only nonzero elements from sparse objects, or
+  //! extract the entire dense object.
+  template<typename MatType>
+  inline arma::Col<typename MatType::elem_type> ExtractNonzeros(
+      const MatType& coordinates) const;
+
+  template<typename eT>
+  inline arma::Col<eT> ExtractNonzeros(const arma::SpMat<eT>& coordinates)
+      const;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "l1_constraint_impl.hpp"
+
+#endif

include/ensmallen_bits/fbs/l1_constraint_impl.hpp

@@ -0,0 +1,201 @@
+/**
+ * @file l1_constraint_impl.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of the proximal operator for the L1 constraint.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_L1_CONSTRAINT_IMPL_HPP
+#define ENSMALLEN_FBS_L1_CONSTRAINT_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "l1_constraint.hpp"
+
+namespace ens {
+
+inline L1Constraint::L1Constraint(const double lambda) : lambda(lambda)
+{
+  // Nothing to do.
+}
+
+template<typename MatType>
+typename MatType::elem_type L1Constraint::Evaluate(const MatType& coordinates)
+    const
+{
+  typedef typename MatType::elem_type eT;
+
+  // Allow some amount of tolerance for floating-point errors.
+  const eT l1Norm = norm(vectorise(coordinates), 1);
+  if (l1Norm <= lambda)
+    return eT(0);
+  else if (std::numeric_limits<eT>::has_infinity)
+    return std::numeric_limits<eT>::infinity();
+  else
+    return std::numeric_limits<eT>::max();
+}
+
+template<typename MatType>
+void L1Constraint::ProximalStep(MatType& coordinates,
+                                const double /* stepSize */)
+    const
+{
+  // First determine whether projection is necessary.
+  if (norm(vectorise(coordinates), 1) <= lambda)
+  {
+    return;
+  }
+
+  // An empty vector can't be projected.
+  if (coordinates.n_elem == 0)
+  {
+    return;
+  }
+
+  // We use the algorithm denoted in Figure 2 of the following paper:
+  //
+  // ```
+  // @inproceedings{duchi2008efficient,
+  //   title={Efficient projections onto the L1-ball for learning in high
+  //       dimensions},
+  //   author={Duchi, John and Shalev-Shwartz, Shai and Singer, Yoram and
+  //       Chandra, Tushar},
+  //   booktitle={Proceedings of the 25th international conference on
+  //       Machine learning},
+  //   pages={272--279},
+  //   year={2008}
+  // }
+  // ```
+  //
+  // This is an iterative algorithm that has a quicksort feel, where we try to
+  // determine the "pivot" element that tells us how much we need to shrink.  In
+  // the original paper, they maintain lists indicating whether a point is above
+  // or below the pivot, but it is more expedient (and efficient) to simply copy
+  // the coordinates array and partially sort it in-place.
+
+  typedef typename MatType::elem_type eT;
+  arma::Col<eT> work = ExtractNonzeros(coordinates);
+  size_t firstUpperElement = 0;
+  size_t lastUpperElement = work.n_elem;
+  eT rho = eT(0); // This is the quantity we aim to find to perform the projection.
+  eT s = eT(0);
+
+  while (lastUpperElement > firstUpperElement)
+  {
+    const size_t k = arma::randi<size_t>(
+        arma::distr_param((int) firstUpperElement, (int) lastUpperElement - 1));
+    const eT v = work[k];
+
+    // Now perform a half-quicksort such that all elements greater than v are in
+    // the first part of the array.
+    size_t left = firstUpperElement;
+    size_t right = lastUpperElement - 1;
+    while (left <= right)
+    {
+      while ((left < lastUpperElement) && (work[left] >= v))
+        ++left;
+      while ((right > firstUpperElement) && (work[right] < v))
+        --right;
+
+      if (left >= right)
+        break;
+
+      // work[left] is less than v, and work[right] is not.  Since we want all
+      // elements greater than or equal to v on the left, swap.
+      const eT tmp = work[left];
+      work[left] = work[right];
+      work[right] = tmp;
+    }
+
+    // Now, work[0] through work[left - 1] are in the greater set G.
+    const eT sDelta = accu(work.subvec(firstUpperElement, left - 1));
+    const size_t rhoDelta = (left - firstUpperElement);
+
+    if ((s + sDelta) - ((eT) (rho + rhoDelta)) * v < eT(lambda))
+    {
+      s += sDelta;
+      rho += rhoDelta;
+      firstUpperElement = left;
+    }
+    else
+    {
+      // v was an element that was less than rho, so, shrink the array and try
+      // again with larger elements.  We actually want to shrink the array so
+      // that it does not include v, so we need to find the first element that
+      // is v (since there may be duplicates).
+      size_t firstVIndex = left - 1;
+      while ((work[firstVIndex] == v) && (firstVIndex >= firstUpperElement))
+        --firstVIndex;
+      lastUpperElement = firstVIndex + 1;
+    }
+  }
+
+  const eT theta = (s - eT(lambda)) / rho;
+  // This is a single-line implementation of the .transform() below; we use the
+  // single-line implementation so it works with Bandicoot.
+  //
+  // coordinates.transform(
+  //     [theta](eT val)
+  //     {
+  //       if (val > 0)
+  //         return std::max(val - theta, eT(0));
+  //       else
+  //         return std::min(val + theta, eT(0));
+  //     });
+  coordinates = sign(coordinates) % clamp(
+      abs(coordinates) - theta, eT(0), std::numeric_limits<eT>::max());
+
+  // Sanity check: ensure we actually ended up inside the L1 ball.  This might
+  // not happen due to floating-point inaccuracies.  If so, try again.
+  const eT newNorm = norm(coordinates, 1);
+  if (newNorm > eT(lambda) && eT(lambda) > eT(0))
+  {
+    // Shrink the L1 ball by the amount of the error.
+    eT newLambda = (eT(lambda) - 2 * (newNorm - eT(lambda)));
+    if (newLambda == eT(lambda))
+    {
+      // Make sure we at least remove a few ULPs.
+      newLambda = eT(lambda) -
+          5 * (eT(lambda) - eT(std::nexttoward(lambda, 0.0)));
+    }
+
+    L1Constraint newConstraint(newLambda);
+    newConstraint.ProximalStep(coordinates, 0.0 /* ignored */);
+  }
+}
+
+// Helper function: extract only nonzero elements from sparse objects, or
+// extract the entire dense object.
+template<typename MatType>
+inline arma::Col<typename MatType::elem_type> L1Constraint::ExtractNonzeros(
+    const MatType& coordinates) const
+{
+  typedef typename MatType::elem_type ElemType;
+  return conv_to<arma::Col<ElemType>>::from(vectorise(abs(coordinates)));
+}
+
+template<typename eT>
+inline arma::Col<eT> L1Constraint::ExtractNonzeros(
+    const arma::SpMat<eT>& coordinates) const
+{
+  arma::Col<eT> result(coordinates.n_nonzero);
+  typename arma::SpMat<eT>::const_iterator it = coordinates.begin();
+  size_t i = 0;
+  while (it != coordinates.end())
+  {
+    // Extract only nonzero values.  Note we use the absolute value because that
+    // is what the algorithm requires (not the original value).
+    result[i] = std::abs(*it);
+    ++it;
+    ++i;
+  }
+
+  return result;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/fbs/l1_penalty.hpp

@@ -0,0 +1,63 @@
+/**
+ * @file l1_penalty.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of the proximal operator for the L1 penalty (also known as
+ * the shrinkage operator).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_L1_PENALTY_HPP
+#define ENSMALLEN_FBS_L1_PENALTY_HPP
+
+namespace ens {
+
+/**
+ * The L1Penalty applies a non-differentiable L1-norm penalty to the coordinates
+ * during optimization:
+ *
+ *   `lambda * || coordinates ||_1`
+ *
+ * This class is meant to be used with the FBS optimizer, and any other
+ * optimizer that uses a proximal operator/step.
+ */
+class L1Penalty
+{
+ public:
+  /**
+   * Construct an L1Penalty object with a given penalty `lambda`.
+   */
+  L1Penalty(const double lambda = 0.0);
+
+  /**
+   * Evaluate the L1 penalty function: `lambda * || coordinates ||_1`.
+   */
+  template<typename MatType>
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const;
+
+  /**
+   * After taking a forward step of size `stepSize`, apply a backwards step /
+   * proximal operator that applies the L1 penalty to `coordinates`.
+   */
+  template<typename MatType>
+  void ProximalStep(MatType& coordinates, const double stepSize) const;
+
+  //! Get the L1 penalty to use when applying the proximal step.
+  double Lambda() const { return lambda; }
+  //! Modify the L1 penalty to use when applying the proximal step.
+  double& Lambda() { return lambda; }
+
+ private:
+  //! The L1 penalty value to use.
+  double lambda;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "l1_penalty_impl.hpp"
+
+#endif

include/ensmallen_bits/fbs/l1_penalty_impl.hpp

@@ -0,0 +1,63 @@
+/**
+ * @file l1_penalty_impl.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of the proximal operator for the L1 penalty (also known as
+ * the shrinkage operator).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FBS_L1_PENALTY_IMPL_HPP
+#define ENSMALLEN_FBS_L1_PENALTY_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "l1_penalty.hpp"
+
+namespace ens {
+
+inline L1Penalty::L1Penalty(const double lambda) : lambda(lambda)
+{
+  // Nothing to do.
+}
+
+template<typename MatType>
+typename MatType::elem_type L1Penalty::Evaluate(const MatType& coordinates)
+    const
+{
+  // Compute the L1 penalty.
+  return norm(vectorise(coordinates), 1) * typename MatType::elem_type(lambda);
+}
+
+template<typename MatType>
+void L1Penalty::ProximalStep(MatType& coordinates,
+                             const double stepSize) const
+{
+  // Apply the backwards step coordinate-wise.  If `MatType` is sparse, this
+  // only applies to nonzero elements, which is just fine.
+  typedef typename MatType::elem_type eT;
+
+  // This is equivalent to the following .transform() implementation (which is
+  // easier to read but will not work with Bandicoot):
+  //
+  //arma::Mat<typename MatType::elem_type> c2 = conv_to<arma::Mat<typename MatType::elem_type>>::from(coordinates);
+  //c2.transform([this, stepSize](eT val) { return (val > eT(0)) ?
+  //    (std::max(eT(0), val - eT(lambda * stepSize))) :
+  //    (std::min(eT(0), val + eT(lambda * stepSize))); });
+  // coordinates.transform([this, stepSize](eT val) { return (val > eT(0)) ?
+  //     (std::max(eT(0), val - eT(lambda * stepSize))) :
+  //     (std::min(eT(0), val + eT(lambda * stepSize))); });
+  //
+  coordinates = sign(coordinates) % clamp(
+      abs(coordinates) - eT(lambda * stepSize), eT(0),
+      std::numeric_limits<eT>::max());
+
+  //coordinates.print("coordinates");
+  //c2.print("c2");
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/fista/fista.hpp

@@ -0,0 +1,214 @@
+/**
+ * @file fista.hpp
+ * @author Ryan Curtin
+ *
+ * An implementation of FISTA (Fast Iterative Shrinkage-Thresholding Algorithm).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FISTA_FISTA_HPP
+#define ENSMALLEN_FISTA_FISTA_HPP
+
+#include "../fbs/l1_penalty.hpp"
+#include "../fbs/l1_constraint.hpp"
+
+namespace ens {
+
+/**
+ * FISTA (Fast Iterative Shrinkage-Thresholding Algorithm) is a proximal
+ * gradient optimization technique for optimizing a function of the form
+ *
+ *   h(x) = f(x) + g(x)
+ *
+ * where f(x) is a differentiable function and g(x) is an arbitrary
+ * non-differentiable function.  In such a situation, standard gradient descent
+ * techniques cannot work because of the non-differentiability of g(x).  To work
+ * around this, FISTA takes a _forward step_ that is just a gradient descent
+ * step on f(x), and then a _backward step_ that is the _proximal operator_
+ * corresponding to g(x).  This continues until convergence.
+ *
+ * This implementation of FISTA allows specification of the backward step (or
+ * proximal operator) via the `BackwardStepType` template parameter.  When using
+ * FBS, the differentiable `FunctionType` given to `Optimize()` should be f(x),
+ * *not* the combined function h(x).  g(x) should be specified by the choice of
+ * `BackwardStepType` (e.g. `L1Penalty` or `L1Maximum`).  The `Optimize()`
+ * function will then return optimized coordinates for h(x), not f(x).
+ *
+ * For more information, see the following paper:
+ *
+ * ```
+ * @article{beck2009fast,
+ *   title={A fast iterative shrinkage-thresholding algorithm for linear inverse
+ *       problems},
+ *   author={Beck, Amir and Teboulle, Marc},
+ *   journal={SIAM Journal On Imaging Sciences},
+ *   volume={2},
+ *   number={1},
+ *   pages={183--202},
+ *   year={2009},
+ *   publisher={SIAM}
+ * }
+ * ```
+ */
+template<typename BackwardStepType = L1Penalty>
+class FISTA
+{
+ public:
+  /**
+   * Construct the FISTA optimizer with the given options, using a
+   * default-constructed BackwardStepType.
+   */
+  FISTA(const size_t maxIterations = 10000,
+        const double tolerance = 1e-10,
+        const size_t maxLineSearchSteps = 50,
+        const double stepSizeAdjustment = 2.0,
+        const bool estimateStepSize = true,
+        const size_t estimateTrials = 10,
+        const double maxStepSize = 0.001);
+
+  /**
+   * Construct the FISTA optimizer with the given options.
+   */
+  FISTA(BackwardStepType backwardStepType,
+        const size_t maxIterations = 10000,
+        const double tolerance = 1e-10,
+        const size_t maxLineSearchSteps = 50,
+        const double stepSizeAdjustment = 2.0,
+        const bool estimateStepSize = true,
+        const size_t estimateTrials = 10,
+        const double maxStepSize = 0.001);
+
+  /**
+   * Optimize the given function using FISTA.  The given starting
+   * point will be modified to store the finishing point of the algorithm,
+   * the final objective value is returned.
+   *
+   * The FunctionType template class must provide the following functions:
+   *
+   *   double Evaluate(const arma::mat& coordinates);
+   *   void Gradient(const arma::mat& coordinates,
+   *                 arma::mat& gradient);
+   *
+   * @tparam FunctionType Type of function to be optimized.
+   * @tparam MatType Type of objective matrix.
+   * @tparam GradType Type of gradient matrix (default is MatType).
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to be optimized.
+   * @param iterate Input with starting point, and will be modified to save
+   *                the output optimial solution coordinates.
+   * @param callbacks Callback functions.
+   * @return Objective value at the final solution.
+   */
+  template<typename FunctionType, typename MatType, typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsMatrixType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the backward step object.
+  const BackwardStepType& BackwardStep() const { return backwardStep; }
+  //! Modify the backward step object.
+  BackwardStepType& BackwardStep() { return backwardStep; }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return maxIterations; }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return maxIterations; }
+
+  //! Get the tolerance on the gradient norm for termination.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance on the gradient norm for termination.
+  double& Tolerance() { return tolerance; }
+
+  //! Get the maximum number of line search steps.
+  size_t MaxLineSearchSteps() const { return maxLineSearchSteps; }
+  //! Modify the maximum number of line search steps.
+  size_t& MaxLineSearchSteps() { return maxLineSearchSteps; }
+
+  //! Get the step size adjustment parameter.
+  double StepSizeAdjustment() const { return stepSizeAdjustment; }
+  //! Modify the step size adjustment parameter.
+  double& StepSizeAdjustment() { return stepSizeAdjustment; }
+
+  //! Get whether or not to estimate the initial step size.
+  bool EstimateStepSize() const { return estimateStepSize; }
+  //! Modify whether or not to estimate the initial step size.
+  bool& EstimateStepSize() { return estimateStepSize; }
+
+  //! Get the number of trials to use for Lipschitz constant estimation.
+  size_t EstimateTrials() const { return estimateTrials; }
+  //! Modify the number of trials to use for Lipschitz constant estimation.
+  size_t& EstimateTrials() { return estimateTrials; }
+
+  //! Get the maximum step size.  If Optimize() has been called, this will
+  //! contain the estimated maximum step size value.
+  double MaxStepSize() const { return maxStepSize; }
+  //! Modify the step size (ignored if EstimateStepSize() is true).
+  double& MaxStepSize() { return maxStepSize; }
+
+ private:
+  //! Utility function: fill with random values.
+  template<typename MatType>
+  static void RandomFill(MatType& x,
+                         const size_t rows,
+                         const size_t cols,
+                         const typename MatType::elem_type maxVal);
+
+  template<typename eT>
+  static void RandomFill(arma::SpMat<eT>& x,
+                         const size_t rows,
+                         const size_t cols,
+                         const eT maxVal);
+
+  template<typename FunctionType, typename MatType>
+  void EstimateLipschitzStepSize(FunctionType& f, const MatType& x);
+
+  //! The instantiated backward step object.
+  BackwardStepType backwardStep;
+
+  //! The maximum number of allowed iterations.
+  size_t maxIterations;
+
+  //! The tolerance for termination.
+  double tolerance;
+
+  //! The maximum number of line search trials.
+  size_t maxLineSearchSteps;
+
+  //! The step size adjustment parameter for the line search.
+  double stepSizeAdjustment;
+
+  //! Whether or not to try and estimate the initial step size.
+  bool estimateStepSize;
+
+  //! Number of trials to use for initial step size estimation.
+  size_t estimateTrials;
+
+  //! The maximum step size to use (estimated if estimateStepSize is true).
+  double maxStepSize;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "fista_impl.hpp"
+
+#endif

include/ensmallen_bits/fista/fista_impl.hpp

@@ -0,0 +1,448 @@
+/**
+ * @file fista_impl.hpp
+ * @author Ryan Curtin
+ *
+ * Implementation of FISTA (Fast Iterative Shrinkage-Thresholding Algorithm).
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FISTA_FISTA_IMPL_HPP
+#define ENSMALLEN_FISTA_FISTA_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "fista.hpp"
+
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+//! Constructor of the FBS class.
+template<typename BackwardStepType>
+FISTA<BackwardStepType>::FISTA(const size_t maxIterations,
+                               const double tolerance,
+                               const size_t maxLineSearchSteps,
+                               const double stepSizeAdjustment,
+                               const bool estimateStepSize,
+                               const size_t estimateTrials,
+                               const double maxStepSize) :
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    maxLineSearchSteps(maxLineSearchSteps),
+    stepSizeAdjustment(stepSizeAdjustment),
+    estimateStepSize(estimateStepSize),
+    estimateTrials(estimateTrials),
+    maxStepSize(maxStepSize)
+{
+  // Check estimateSteps parameter.
+  if (estimateStepSize && estimateTrials == 0)
+  {
+    throw std::invalid_argument("FISTA::FISTA(): estimateTrials must be greater"
+        " than 0!");
+  }
+}
+
+template<typename BackwardStepType>
+FISTA<BackwardStepType>::FISTA(BackwardStepType backwardStep,
+                               const size_t maxIterations,
+                               const double tolerance,
+                               const size_t maxLineSearchSteps,
+                               const double stepSizeAdjustment,
+                               const bool estimateStepSize,
+                               const size_t estimateTrials,
+                               const double maxStepSize) :
+    backwardStep(std::move(backwardStep)),
+    maxIterations(maxIterations),
+    tolerance(tolerance),
+    maxLineSearchSteps(maxLineSearchSteps),
+    stepSizeAdjustment(stepSizeAdjustment),
+    estimateStepSize(estimateStepSize),
+    estimateTrials(estimateTrials),
+    maxStepSize(maxStepSize)
+{
+  // Check estimateSteps parameter.
+  if (estimateStepSize && estimateTrials == 0)
+  {
+    throw std::invalid_argument("FISTA::FISTA(): estimateTrials must be greater"
+        " than 0!");
+  }
+}
+
+//! Optimize the function (minimize).
+template<typename BackwardStepType>
+template<typename FunctionType, typename MatType, typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
+FISTA<BackwardStepType>::Optimize(FunctionType& function,
+                                  MatType& iterateIn,
+                                  CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
+  FullFunctionType& f = static_cast<FullFunctionType&>(function);
+
+  // Make sure we have all necessary functions.
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  // Match the notation of the paper.  We force a copy here, since we use
+  // std::move() internally and this may be an alias.  We copy back to
+  // `iterateIn` at the end.
+  BaseMatType x(iterateIn);
+
+  // To keep track of the function value.
+  ElemType lastObj = std::numeric_limits<ElemType>::max();;
+  ElemType currentFObj = f.Evaluate(x);
+  ElemType currentGObj = backwardStep.Evaluate(x);
+  ElemType currentObj = currentFObj + currentGObj;
+
+  BaseGradType g(x.n_rows, x.n_cols); // Gradient.
+  BaseMatType y = x; // Initialize y_1 = x_0.
+  BaseMatType lastX;
+  ElemType t = 1; // Initialize t_1 = 1.
+  ElemType lastT = t;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // First, estimate the Lipschitz constant to set the initial/maximum step
+  // size, if the user asked us to.
+  if (estimateStepSize)
+    EstimateLipschitzStepSize(f, x); // Sets `maxStepSize`.
+
+  // Keep track of the last step size we used.
+  ElemType currentStepSize = (ElemType) maxStepSize;
+  ElemType lastStepSize = (ElemType) maxStepSize;
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
+  Callback::BeginOptimization(*this, f, x, callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
+  {
+    // During this optimization, we want to optimize h(x) = f(x) + g(x).
+    // f(x) is `f`, but g(x) is specified by `BackwardStepType`.
+
+    // Notation (compare with Beck and Teboulle):
+    //   `i` represents `k`, the iteration number.
+    //   `x` represents `x_k` in the paper.
+    //   `y` represents `y_k` in the paper.
+
+    // The first step is to compute a step size via a line search.  To do this,
+    // we need to compute the gradient f'(y) as required by the quadratic
+    // approximation Q_L(x, y) (Eq. 2.5).
+    //
+    // We will also need the objective f(y), so we will compute that
+    // simultaneously.
+    const ElemType yObj = f.EvaluateWithGradient(y, g);
+    terminate |= Callback::EvaluateWithGradient(*this, f, y, yObj, g,
+        callbacks...);
+
+    // Use backtracking line search to find the best step size.  This is not the
+    // version from the FASTA paper (non-monotone line search) but instead the
+    // version proposed by Beck and Teboulle, with a minor modification: we
+    // start our search at the last step size, and allow the search to increase
+    // the step size up to the maximum step size if it can.  This is a more
+    // effective heuristic than simply starting at the largest allowable step
+    // size and shrinking from there, especially in regions where the gradient
+    // norm is small.  It is also more effective than simply starting at the
+    // last step size and shrinking from there, as it prevents getting "stuck"
+    // with a very small step size.
+    bool lsDone = false;
+    size_t lsTrial = 0;
+    bool increasing = false; // Will be set during the first iteration.
+    ElemType lastFObj = ElemType(0);
+    ElemType lastGObj = ElemType(0);
+    BaseMatType lsLastX; // Only used in increasing mode.
+    BaseMatType xDiff;
+
+    lastX = std::move(x);
+    lastStepSize = currentStepSize;
+    currentStepSize = std::min(currentStepSize, (ElemType) maxStepSize);
+
+    while (!lsDone && !terminate)
+    {
+      if (lsTrial == maxLineSearchSteps)
+      {
+        if (increasing)
+        {
+          Warn << "FISTA::Optimize(): line search reached maximum number of "
+              << "steps (" << maxLineSearchSteps << "); using step size "
+              << currentStepSize << "." << std::endl;
+          break; // The step size is still valid.
+        }
+        else
+        {
+          Warn << "FISTA::Optimize(): could not find valid step size in range "
+              << "(0, " << maxStepSize << "]!  Terminating optimization."
+              << std::endl;
+          x = std::move(lastX); // Revert to previous coordinates.
+          terminate = true;
+          break;
+        }
+      }
+
+      // If the step size has converged to zero, we are done.
+      if (currentStepSize == ElemType(0))
+      {
+        Warn << "FISTA::Optimize(): computed zero step size; terminating "
+            << "optimization." << std::endl;
+        x = std::move(lastX); // Revert to previous coordinates.
+        terminate = true;
+        break;
+      }
+
+      // Perform forward update into x.
+      x = y - currentStepSize * g;
+      backwardStep.ProximalStep(x, currentStepSize);
+
+      // Compute F(x) = f(x) + g(x).
+      const ElemType fObj = f.Evaluate(x);
+      const ElemType gObj = backwardStep.Evaluate(x);
+      const ElemType lsObj = fObj + gObj;
+      terminate |= Callback::Evaluate(*this, f, x, fObj, callbacks...);
+
+      // Compute Q_L(x, y) (the quadratic approximation), Eq. (2.5).
+      xDiff = x - y;
+      const ElemType q = yObj + dot(xDiff, g) +
+          (1 / (2 * currentStepSize)) * dot(xDiff, xDiff) + gObj;
+
+      // If we're on the first iteration, we don't know if we should be
+      // searching for a step size by increasing or decreasing the step size.
+      // (Remember that our valid ranges of step sizes are [0, maxStepSize], and
+      // we are starting at lastStepSize.)
+      //
+      // Thus, if the condition is satisfied, let's try increasing the step size
+      // until it's no longer satisfied.  Otherwise, we will have to decrease
+      // the step size.
+      if (lsTrial == 0)
+      {
+        increasing = (lsObj <= q);
+      }
+
+      if (increasing)
+      {
+        // If we are in "increasing" mode, then termination occurs on the first
+        // iteration when the condition is *not* satisfied (and we use the last
+        // step size).
+        if ((lsObj > q) || (!std::isfinite(lsObj)))
+        {
+          lsDone = true;
+          if (lsTrial != 0)
+            x = std::move(lsLastX);
+          currentFObj = lastFObj;
+          currentGObj = lastGObj;
+          lastObj = currentObj;
+          currentObj = currentFObj + currentGObj;
+          currentStepSize = lastStepSize; // Take one step backwards.
+        }
+        else if (currentStepSize == (ElemType) maxStepSize)
+        {
+          // The condition is still satisfied, but the step size will be too big
+          // if we take another step.  Go back to the maximum step size.
+          lsDone = true;
+          currentFObj = fObj;
+          currentGObj = gObj;
+          lastObj = currentObj;
+          currentObj = currentFObj + currentGObj;
+        }
+        else
+        {
+          // The condition is still satisfied; increase the step size.
+          lastStepSize = currentStepSize;
+          currentStepSize *= ElemType(stepSizeAdjustment);
+          lsLastX = std::move(x);
+          lastFObj = fObj;
+          lastGObj = gObj;
+          ++lsTrial;
+        }
+      }
+      else
+      {
+        // If we are in "decreasing" mode, then termination occurs on the first
+        // iteration when the condition is satisfied.
+        if ((lsObj <= q) && (std::isfinite(lsObj)))
+        {
+          lsDone = true;
+          currentFObj = fObj;
+          currentGObj = gObj;
+          lastObj = currentObj;
+          currentObj = currentFObj + currentGObj;
+        }
+        else
+        {
+          // The condition is not yet satisfied; decrease the step size.
+          currentStepSize /= ElemType(stepSizeAdjustment);
+          ++lsTrial;
+        }
+      }
+    }
+
+    // If we terminated during the line search, we are done.
+    if (terminate)
+      break;
+
+    if (!lsDone)
+    {
+      // The line search failed, so terminate.
+      Warn << "FISTA::Optimize(): line search failed after "
+          << maxLineSearchSteps << " steps; terminating optimization."
+          << std::endl;
+      x = std::move(lastX);
+      terminate = true;
+      break;
+    }
+
+    // Output current objective function.
+    Info << "FISTA::Optimize(): iteration " << i << ", combined objective "
+        << currentObj << " (f(x) = " << currentFObj << ", g(x) = "
+        << currentGObj << "), step size " << currentStepSize << "."
+        << std::endl;
+
+    if ((i > 1) && !std::isfinite(currentObj))
+    {
+      Warn << "FISTA::Optimize(): objective diverged to " << currentObj
+          << "; terminating optimization." << std::endl;
+      terminate = true;
+      break;
+    }
+
+    // Check for convergence.  This is a simple check on the objective.
+    if ((i > 1) && (std::abs(currentObj - lastObj) < tolerance))
+    {
+      Info << "FISTA::Optimize(): minimized within objective tolerance "
+          << tolerance << "; terminating optimization." << std::endl;
+      terminate = true;
+    }
+
+    // Compute updated prediction parameter t.
+    lastT = t;
+    t = (1 + std::sqrt(1 + 4 * std::pow(t, ElemType(2)))) / 2;
+
+    // Sometimes t can get to be too large; this restart scheme is taken
+    // originally from O'Donoghue and Candes, "Adaptive restart for accelerated
+    // gradient schemes", 2012.
+    const ElemType restartCheck = dot(y - x, x - lastX);
+    if (restartCheck > 0)
+    {
+      Info << "FISTA::Optimize(): t too large (" << t << "); reset to 1."
+          << std::endl;
+      t = 1;
+      lastT = 1;
+    }
+
+    // Update prediction y.
+    y = x + ((lastT - 1) / t) * (x - lastX);
+
+    terminate |= Callback::StepTaken(*this, f, y, callbacks...);
+  }
+
+  if (!terminate)
+  {
+    Info << "FISTA::Optimize(): maximum iterations (" << maxIterations
+        << ") reached; terminating optimization." << std::endl;
+  }
+
+  Callback::EndOptimization(*this, f, x, callbacks...);
+
+  ((BaseMatType&) iterateIn) = x;
+  return currentObj;
+} // Optimize()
+
+template<typename BackwardStepType>
+template<typename MatType>
+void FISTA<BackwardStepType>::RandomFill(
+    MatType& x,
+    const size_t rows,
+    const size_t cols,
+    const typename MatType::elem_type maxVal)
+{
+  x.randu(rows, cols);
+  x *= maxVal;
+}
+
+template<typename BackwardStepType>
+template<typename eT>
+void FISTA<BackwardStepType>::RandomFill(
+    arma::SpMat<eT>& x,
+    const size_t rows,
+    const size_t cols,
+    const eT maxVal)
+{
+  eT density = eT(0.1);
+  // Try and keep the matrix from having too many elements.
+  if (rows * cols > 100000)
+    density = eT(0.01);
+  else if (rows * cols > 1000000)
+    density = eT(0.001);
+  else if (rows * cols > 10000000)
+    density = eT(0.0001);
+
+  x.sprandu(rows, cols, density);
+
+  // Make sure we got at least some nonzero elements...
+  while (x.n_nonzero == 0)
+  {
+    if (x.n_elem < 10)
+      x.sprandu(rows, cols, 1.0);
+    else
+      x.sprandu(rows, cols, 0.5);
+  }
+
+  x *= maxVal;
+}
+
+template<typename BackwardStepType>
+template<typename FunctionType, typename MatType>
+void FISTA<BackwardStepType>::EstimateLipschitzStepSize(
+    FunctionType& f,
+    const MatType& x)
+{
+  typedef typename MatType::elem_type ElemType;
+
+  // Sanity check for estimateSteps parameter.
+  if (estimateTrials == 0)
+  {
+    throw std::invalid_argument("FISTA::Optimize(): estimateTrials must be "
+        "greater than 0!");
+  }
+
+  const ElemType xMax = std::max(ElemType(1), 2 * x.max());
+  ElemType sum = ElemType(0);
+  MatType x1, x2, gx1, gx2;
+
+  for (size_t t = 0; t < estimateTrials; ++t)
+  {
+    RandomFill(x1, x.n_rows, x.n_cols, xMax);
+    RandomFill(x2, x.n_rows, x.n_cols, xMax);
+
+    f.Gradient(x1, gx1);
+    f.Gradient(x2, gx2);
+
+    // Compute a Lipschitz constant estimate.
+    const ElemType lEst = norm(gx1 - gx2, 2) / norm(x1 - x2, 2);
+    sum += lEst;
+  }
+
+  sum /= estimateTrials;
+  if (sum == 0)
+    maxStepSize = std::numeric_limits<ElemType>::max();
+  else
+    maxStepSize = (10 / sum);
+
+  Info << "FISTA::Optimize(): estimated a maximum step size of "
+      << maxStepSize << "." << std::endl;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ftml/ftml.hpp

@@ -98,7 +98,7 @@ class FTML
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/ftml/ftml_update.hpp

@@ -78,6 +78,8 @@ class FTMLUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -87,11 +89,18 @@ class FTMLUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(FTMLUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent)
+        parent(parent),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2))
     {
       v.zeros(rows, cols);
       z.zeros(rows, cols);
       d.zeros(rows, cols);
+
+      // Attempt to catch underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -109,19 +118,19 @@ class FTMLUpdate
       ++iteration;
 
       // And update the iterate.
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
-      MatType sigma = -parent.beta1 * d;
-      d = biasCorrection1 / stepSize *
-        (arma::sqrt(v / biasCorrection2) + parent.epsilon);
+      MatType sigma = -beta1 * d;
+      d = biasCorrection1 / ElemType(stepSize) *
+          (sqrt(v / biasCorrection2) + epsilon);
       sigma += d;
 
-      z *= parent.beta1;
-      z += (1 - parent.beta1) * gradient - sigma % iterate;
+      z *= beta1;
+      z += (1 - beta1) * gradient - sigma % iterate;
       iterate = -z / d;
     }
 
@@ -140,6 +149,11 @@ class FTMLUpdate
 
     // The number of iterations.
     size_t iteration;
+
+    // Optimization parameters converted to the type of the optimization.
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
   };
 
  private:

include/ensmallen_bits/function/arma_traits.hpp

@@ -122,6 +122,17 @@ template<>
 inline void RequireDenseFloatingPointType<arma::mat>() { }
 template<>
 inline void RequireDenseFloatingPointType<arma::fmat>() { }
+#if defined(ARMA_HAVE_FP16)
+template<>
+inline void RequireDenseFloatingPointType<arma::hmat>() { }
+#endif
+
+#ifdef ENS_HAVE_COOT
+template<>
+inline void RequireDenseFloatingPointType<coot::mat>() { }
+template<>
+inline void RequireDenseFloatingPointType<coot::fmat>() { }
+#endif
 
 template<typename MatType>
 void RequireFloatingPointType()
@@ -144,6 +155,19 @@ template<>
 inline void RequireFloatingPointType<arma::sp_mat>() { }
 template<>
 inline void RequireFloatingPointType<arma::sp_fmat>() { }
+#if defined(ARMA_HAVE_FP16)
+template<>
+inline void RequireFloatingPointType<arma::hmat>() { }
+template<>
+inline void RequireFloatingPointType<arma::sp_hmat>() { }
+#endif
+
+#ifdef ENS_HAVE_COOT
+template<>
+inline void RequireFloatingPointType<coot::mat>() { }
+template<>
+inline void RequireFloatingPointType<coot::fmat>() { }
+#endif
 
 /**
  * Require that the internal element type of the matrix type and gradient type

include/ensmallen_bits/fw/atoms.hpp

@@ -96,6 +96,7 @@ class Atoms
       // Find possible atom to be deleted.
       arma::vec gap = sqTerm -
           currentCoeffs % trans(gradient.t() * currentAtoms);
+
       arma::uword ind = gap.index_min();
 
       // Try deleting the atom.

include/ensmallen_bits/fw/constr_lpball.hpp

@@ -49,7 +49,8 @@ namespace ens {
  * \f]
  *
  */
-class ConstrLpBallSolver
+template<typename VecType = arma::vec>
+class ConstrLpBallSolverType
 {
  public:
   /**
@@ -58,7 +59,7 @@ class ConstrLpBallSolver
    *
    * @param p The constraint is unit lp ball.
    */
-  ConstrLpBallSolver(const double p) : p(p)
+  ConstrLpBallSolverType(const double p) : p(p)
   { /* Do nothing. */ }
 
   /**
@@ -68,7 +69,7 @@ class ConstrLpBallSolver
    * @param p The constraint is unit lp ball.
    * @param lambda Regularization parameter.
    */
-  ConstrLpBallSolver(const double p, const arma::vec lambda) :
+  ConstrLpBallSolverType(const double p, const VecType lambda) :
       p(p), regFlag(true), lambda(lambda)
   { /* Do nothing. */ }
 
@@ -80,52 +81,51 @@ class ConstrLpBallSolver
    * @param s Output optimal solution in the constrained domain (lp ball).
    */
   template<typename MatType>
-  void Optimize(const MatType& v,
-                MatType& s)
+  void Optimize(const MatType& v, MatType& s)
   {
-    typedef typename MatType::elem_type ElemType;
+    typedef typename ForwardType<MatType>::uword UWordType;
 
     if (p == std::numeric_limits<double>::infinity())
     {
       // l-inf ball.
-      s = -arma::sign(v);
+      s = -sign(v);
       if (regFlag)
       {
         // Do element-wise division.
-        s /= arma::conv_to<arma::Col<ElemType>>::from(lambda);
+        s /= conv_to<MatType>::from(lambda);
       }
     }
     else if (p > 1.0)
     {
       // lp ball with 1<p<inf.
       if (regFlag)
-        s = v / arma::conv_to<arma::Col<ElemType>>::from(lambda);
+        s = v / conv_to<MatType>::from(lambda);
       else
         s = v;
 
       double q = 1 / (1.0 - 1.0 / p);
-      s = -arma::sign(v) % arma::pow(arma::abs(s), q - 1);
-      s = arma::normalise(s, p);
+      s = -sign(v) % pow(abs(s), q - 1);
+      s = normalise(s, p);
 
       if (regFlag)
-        s = s / arma::conv_to<arma::Col<ElemType>>::from(lambda);
+        s = s / conv_to<MatType>::from(lambda);
     }
     else if (p == 1.0)
     {
       // l1 ball, also used in OMP.
       if (regFlag)
-        s = arma::abs(v / arma::conv_to<arma::Col<ElemType>>::from(lambda));
+        s = abs(v / conv_to<MatType>::from(lambda));
       else
-        s = arma::abs(v);
+        s = abs(v);
 
       // k is the linear index of the largest element.
-      arma::uword k = s.index_max();
+      UWordType k = s.index_max();
       s.zeros();
       // Take the sign of v(k).
       s(k) = -((0.0 < v(k)) - (v(k) < 0.0));
 
       if (regFlag)
-        s = s / arma::conv_to<arma::Col<ElemType>>::from(lambda);
+        s = s / conv_to<MatType>::from(lambda);
     }
     else
     {
@@ -146,9 +146,9 @@ class ConstrLpBallSolver
   bool& RegFlag() { return regFlag; }
 
   //! Get the regularization parameter.
-  arma::vec Lambda() const { return lambda; }
+  VecType Lambda() const { return lambda; }
   //! Modify the regularization parameter.
-  arma::vec& Lambda() { return lambda; }
+  VecType& Lambda() { return lambda; }
 
  private:
   //! lp norm, 1<=p<=inf;
@@ -159,9 +159,11 @@ class ConstrLpBallSolver
   bool regFlag = false;
 
   //! Regularization parameter.
-  arma::vec lambda;
+  VecType lambda;
 };
 
+using ConstrLpBallSolver = ConstrLpBallSolverType<arma::vec>;
+
 } // namespace ens
 
 #endif

include/ensmallen_bits/fw/frank_wolfe.hpp

@@ -126,7 +126,7 @@ class FrankWolfe
    */
   template<typename FunctionType, typename MatType, typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(FunctionType& function,
            MatType& iterate,

include/ensmallen_bits/fw/frank_wolfe_impl.hpp

@@ -41,12 +41,12 @@ template<
     typename UpdateRuleType>
 template<typename FunctionType, typename MatType, typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 FrankWolfe<LinearConstrSolverType, UpdateRuleType>::Optimize(
-  FunctionType& function,
-  MatType& iterateIn,
-  CallbackTypes&&... callbacks)
+    FunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
 {
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
@@ -75,8 +75,11 @@ FrankWolfe<LinearConstrSolverType, UpdateRuleType>::Optimize(
   // Controls early termination of the optimization process.
   bool terminate = false;
 
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
   Callback::BeginOptimization(*this, f, iterate, callbacks...);
-  for (size_t i = 1; i != maxIterations && !terminate; ++i)
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     currentObjective = f.EvaluateWithGradient(iterate, gradient);
 
@@ -95,7 +98,7 @@ FrankWolfe<LinearConstrSolverType, UpdateRuleType>::Optimize(
     if (gap < tolerance)
     {
       Info << "FrankWolfe::Optimize(): minimized within tolerance "
-          << tolerance << "; " << "terminating optimization." << std::endl;
+          << tolerance << "; terminating optimization." << std::endl;
 
       Callback::EndOptimization(*this, f, iterate, callbacks...);
       return currentObjective;
@@ -109,8 +112,11 @@ FrankWolfe<LinearConstrSolverType, UpdateRuleType>::Optimize(
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
   }
 
-  Info << "FrankWolfe::Optimize(): maximum iterations (" << maxIterations
-      << ") reached; " << "terminating optimization." << std::endl;
+  if (!terminate)
+  {
+    Info << "FrankWolfe::Optimize(): maximum iterations (" << maxIterations
+        << ") reached; terminating optimization." << std::endl;
+  }
 
   Callback::EndOptimization(*this, f, iterate, callbacks...);
   return currentObjective;

include/ensmallen_bits/fw/line_search/line_search_impl.hpp

@@ -106,7 +106,7 @@ typename MatType::elem_type LineSearch::Derivative(FunctionType& function,
 {
   GradType gradient(x0.n_rows, x0.n_cols);
   function.Gradient(x0 + gamma * deltaX, gradient);
-  return arma::dot(gradient, deltaX);
+  return dot(gradient, deltaX);
 }
 
 } // namespace ens

include/ensmallen_bits/fw/proximal/proximal_impl.hpp

@@ -35,14 +35,24 @@ namespace ens {
 template<typename MatType>
 inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
 {
-  MatType simplexSol = arma::abs(v);
+  MatType simplexSol = abs(v);
 
   // Already with L1 norm <= tau.
-  if (arma::accu(simplexSol) <= tau)
+  if (accu(simplexSol) <= tau)
     return;
 
-  simplexSol = arma::sort(simplexSol, "descend");
-  MatType simplexSum = arma::cumsum(simplexSol);
+  simplexSol = sort(simplexSol, "descend");
+  // MatType simplexSum = arma::cumsum(simplexSol);
+  MatType simplexSum(simplexSol.n_rows, simplexSol.n_cols);
+  for (size_t col = 0; col < simplexSol.n_cols; ++col)
+  {
+    simplexSum(0, col) = simplexSol(0, col);
+    for (size_t row = 1; row < simplexSol.n_rows; ++row)
+    {
+      simplexSum(row, col) = simplexSum(row - 1, col) +
+          simplexSol(row, col);
+    }
+  }
 
   double nu = 0;
   size_t rho = simplexSol.n_rows - 1;
@@ -72,10 +82,15 @@ inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
 template<typename MatType>
 inline void Proximal::ProjectToL0Ball(MatType& v, int tau)
 {
-  arma::uvec indices = arma::sort_index(arma::abs(v));
-  arma::uword numberToKill = v.n_elem - tau;
+  typedef typename ForwardType<MatType>::uword UWordType;
+  typedef typename ForwardType<MatType>::uvec UVecType;
+  typedef typename ForwardType<MatType>::bvec VecType;
 
-  for (arma::uword i = 0; i < numberToKill; i++)
+  const VecType vTemp = conv_to<VecType>::from(abs(v));
+  UVecType indices = sort_index(vTemp);
+  UWordType numberToKill = v.n_elem - tau;
+
+  for (UWordType i = 0; i < numberToKill; i++)
     v(indices(i)) = 0.0;
 }
 

include/ensmallen_bits/fw/update_full_correction.hpp

@@ -78,7 +78,7 @@ class UpdateFullCorrection
     atoms.ProjectedGradientEnhancement(function, tau, stepSize);
     arma::mat tmp;
     atoms.RecoverVector(tmp);
-    newCoords = arma::conv_to<MatType>::from(tmp);
+    newCoords = conv_to<MatType>::from(tmp);
   }
 
  private:

include/ensmallen_bits/fw/update_span.hpp

@@ -63,7 +63,7 @@ class UpdateSpan
     // to the original size.
     arma::mat tmp;
     atoms.RecoverVector(tmp);
-    newCoords = arma::conv_to<MatType>::from(tmp);
+    newCoords = conv_to<MatType>::from(tmp);
 
     // Prune the support.
     if (isPrune)
@@ -72,7 +72,7 @@ class UpdateSpan
       double F = 0.25 * oldF + 0.75 * function.Evaluate(newCoords);
       atoms.PruneSupport(F, function);
       atoms.RecoverVector(tmp);
-      newCoords = arma::conv_to<MatType>::from(tmp);
+      newCoords = conv_to<MatType>::from(tmp);
     }
   }
 

include/ensmallen_bits/gradient_descent/gradient_descent.hpp

@@ -39,8 +39,17 @@ namespace ens {
  * GradientDescent can optimize differentiable functions.  For more details, see
  * the documentation on function types included with this distribution or on the
  * ensmallen website.
+ *
+ * @tparam UpdatePolicyType Update policy used by Gradient Descent during the
+ *     iterative update process. By default vanilla update policy (see
+ *     ens::VanillaUpdate) is used.
+ * @tparam DecayPolicyType Decay policy used during the iterative update
+ *     process to adjust the step size. By default the step size isn't going to
+ *     be adjusted (i.e. NoDecay is used).
  */
-class GradientDescent
+template<typename UpdatePolicyType = VanillaUpdate,
+         typename DecayPolicyType = NoDecay>
+class GradientDescentType
 {
  public:
   /**
@@ -54,10 +63,24 @@ class GradientDescent
    * @param maxIterations Maximum number of iterations allowed (0 means no
    *     limit).
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param updatePolicy Instantiated update policy used to adjust the given
+   *                     parameters.
+   * @param decayPolicy Instantiated decay policy used to adjust the step size.
+   * @param resetPolicy Flag that determines whether update policy parameters
+   *                    are reset before every Optimize call.
    */
-  GradientDescent(const double stepSize = 0.01,
-                  const size_t maxIterations = 100000,
-                  const double tolerance = 1e-5);
+  GradientDescentType(
+      const double stepSize = 0.01,
+      const size_t maxIterations = 100000,
+      const double tolerance = 1e-5,
+      const UpdatePolicyType& updatePolicy = UpdatePolicyType(),
+      const DecayPolicyType& decayPolicy = DecayPolicyType(),
+      const bool resetPolicy = true);
+
+  /**
+   * Clean any memory associated with the GradientDescent object.
+   */
+  ~GradientDescentType();
 
   /**
    * Optimize the given function using gradient descent.  The given starting
@@ -77,7 +100,7 @@ class GradientDescent
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(FunctionType& function,
            MatType& iterate,
@@ -140,9 +163,9 @@ class GradientDescent
       const arma::Row<size_t>& numCategories,
       CallbackTypes&&... callbacks)
   {
-    return Optimize<FunctionType, MatType, MatType,
-        CallbackTypes...>(function, iterate, categoricalDimensions,
-        numCategories, std::forward<CallbackTypes>(callbacks)...);
+    return Optimize<FunctionType, MatType, MatType, CallbackTypes...>(function,
+        iterate, categoricalDimensions, numCategories,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -160,6 +183,37 @@ class GradientDescent
   //! Modify the tolerance for termination.
   double& Tolerance() { return tolerance; }
 
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return resetPolicy; }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return resetPolicy; }
+
+  //! Get the update policy.
+  const UpdatePolicyType& UpdatePolicy() const { return updatePolicy; }
+  //! Modify the update policy.
+  UpdatePolicyType& UpdatePolicy() { return updatePolicy; }
+
+  //! Get the instantiated update policy type.
+  //! Be sure to check its type with Has() before using!
+  const Any& InstUpdatePolicy() const { return instUpdatePolicy; }
+  //! Modify the instantiated update policy type.
+  //! Be sure to check its type with Has() before using!
+  Any& InstUpdatePolicy() { return instUpdatePolicy; }
+
+  //! Get the step size decay policy.
+  const DecayPolicyType& DecayPolicy() const { return decayPolicy; }
+  //! Modify the step size decay policy.
+  DecayPolicyType& DecayPolicy() { return decayPolicy; }
+
+  //! Get the instantiated decay policy type.
+  //! Be sure to check its type with Has() before using!
+  const Any& InstDecayPolicy() const { return instDecayPolicy; }
+  //! Modify the instantiated decay policy type.
+  //! Be sure to check its type with Has() before using!
+  Any& InstDecayPolicy() { return instDecayPolicy; }
+
  private:
   //! The step size for each example.
   double stepSize;
@@ -169,8 +223,30 @@ class GradientDescent
 
   //! The tolerance for termination.
   double tolerance;
+
+  //! The update policy used to update the parameters in each iteration.
+  UpdatePolicyType updatePolicy;
+
+  //! The decay policy used to update the step size.
+  DecayPolicyType decayPolicy;
+
+  //! Flag indicating whether update policy
+  //! should be reset before running optimization.
+  bool resetPolicy;
+
+  //! Flag indicating whether the update policy
+  //! parameters have been initialized.
+  bool isInitialized;
+
+  //! The initialized update policy.
+  Any instUpdatePolicy;
+
+  //! The initialized decay policy.
+  Any instDecayPolicy;
 };
 
+using GradientDescent = GradientDescentType<VanillaUpdate, NoDecay>;
+
 } // namespace ens
 
 #include "gradient_descent_impl.hpp"

include/ensmallen_bits/gradient_descent/gradient_descent_impl.hpp

@@ -20,25 +20,43 @@
 namespace ens {
 
 //! Constructor.
-inline GradientDescent::GradientDescent(
+template <typename UpdatePolicyType, typename DecayPolicyType>
+GradientDescentType<UpdatePolicyType, DecayPolicyType>::GradientDescentType(
     const double stepSize,
     const size_t maxIterations,
-    const double tolerance) :
+    const double tolerance,
+    const UpdatePolicyType& updatePolicy,
+    const DecayPolicyType& decayPolicy,
+    const bool resetPolicy) :
     stepSize(stepSize),
     maxIterations(maxIterations),
-    tolerance(tolerance)
+    tolerance(tolerance),
+    updatePolicy(updatePolicy),
+    decayPolicy(decayPolicy),
+    resetPolicy(resetPolicy),
+    isInitialized(false)
 { /* Nothing to do. */ }
 
+template <typename UpdatePolicyType, typename DecayPolicyType>
+GradientDescentType<UpdatePolicyType, DecayPolicyType>::~GradientDescentType()
+{
+  // Clean decay and update policies, if they were initialized.
+  instDecayPolicy.Clean();
+  instUpdatePolicy.Clean();
+}
+
 //! Optimize the function (minimize).
+template <typename UpdatePolicyType, typename DecayPolicyType>
 template<typename FunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
-GradientDescent::Optimize(FunctionType& function,
-                          MatType& iterateIn,
-                          CallbackTypes&&... callbacks)
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
+GradientDescentType<UpdatePolicyType, DecayPolicyType>::Optimize(
+    FunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
 {
   // Convenience typedefs.
   typedef typename MatType::elem_type ElemType;
@@ -49,6 +67,13 @@ GradientDescent::Optimize(FunctionType& function,
   typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
   FullFunctionType& f(static_cast<FullFunctionType&>(function));
 
+  // The update policy and decay policy internally use a templated class so
+  // that we can know MatType and GradType only when Optimize() is called.
+  typedef typename UpdatePolicyType::template Policy<BaseMatType, BaseGradType>
+      InstUpdatePolicyType;
+  typedef typename DecayPolicyType::template Policy<BaseMatType, BaseGradType>
+      InstDecayPolicyType;
+
   // Make sure we have the methods that we need.
   traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
   RequireFloatingPointType<BaseMatType>();
@@ -65,9 +90,30 @@ GradientDescent::Optimize(FunctionType& function,
   // Controls early termination of the optimization process.
   bool terminate = false;
 
+  // Initialize the decay policy if needed.
+  if (!isInitialized || !instDecayPolicy.Has<InstDecayPolicyType>())
+  {
+    instDecayPolicy.Clean();
+    instDecayPolicy.Set<InstDecayPolicyType>(
+        new InstDecayPolicyType(decayPolicy));
+  }
+
+  // Initialize the update policy.
+  if (resetPolicy || !isInitialized ||
+      !instUpdatePolicy.Has<InstUpdatePolicyType>())
+  {
+    instUpdatePolicy.Clean();
+    instUpdatePolicy.Set<InstUpdatePolicyType>(new InstUpdatePolicyType(
+        updatePolicy, iterate.n_rows, iterate.n_cols));
+    isInitialized = true;
+  }
+
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
   // Now iterate!
   Callback::BeginOptimization(*this, f, iterate, callbacks...);
-  for (size_t i = 1; i != maxIterations && !terminate; ++i)
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     overallObjective = f.EvaluateWithGradient(iterate, gradient);
 
@@ -97,28 +143,40 @@ GradientDescent::Optimize(FunctionType& function,
       return overallObjective;
     }
 
+    // Use the update policy to take a step.
+    instUpdatePolicy.As<InstUpdatePolicyType>().Update(iterate,
+                                                       stepSize,
+                                                       gradient);
+
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
+
+    // Now update the learning rate if requested by the user.
+    instDecayPolicy.As<InstDecayPolicyType>().Update(iterate,
+                                                     stepSize,
+                                                     gradient);
+
     // Reset the counter variables.
     lastObjective = overallObjective;
-
-    // And update the iterate.
-    iterate -= stepSize * gradient;
-    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
   }
 
-  Info << "Gradient Descent: maximum iterations (" << maxIterations
-      << ") reached; " << "terminating optimization." << std::endl;
+  if (!terminate)
+  {
+    Info << "Gradient Descent: maximum iterations (" << maxIterations
+        << ") reached; " << "terminating optimization." << std::endl;
+  }
 
   Callback::EndOptimization(*this, f, iterate, callbacks...);
   return overallObjective;
 }
 
+template <typename UpdatePolicyType, typename DecayPolicyType>
 template<typename FunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
 typename std::enable_if<IsArmaType<GradType>::value,
 typename MatType::elem_type>::type
-GradientDescent::Optimize(
+GradientDescentType<UpdatePolicyType, DecayPolicyType>::Optimize(
     FunctionType& function,
     MatType& iterate,
     const std::vector<bool>& categoricalDimensions,
@@ -159,4 +217,4 @@ GradientDescent::Optimize(
 
 } // namespace ens
 
-#endif
+#endif // ENSMALLEN_GRADIENT_DESCENT_GRADIENT_DESCENT_IMPL_HPP

include/ensmallen_bits/iqn/iqn.hpp

@@ -87,11 +87,11 @@ class IQN
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
-                                       MatType& iterate,
-                                       CallbackTypes&&... callbacks);
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
 
   //! Forward the MatType as GradType.
   template<typename SeparableFunctionType,

include/ensmallen_bits/iqn/iqn_impl.hpp

@@ -36,8 +36,8 @@ template<typename SeparableFunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 IQN::Optimize(SeparableFunctionType& functionIn,
               MatType& iterateIn,
               CallbackTypes&&... callbacks)
@@ -46,6 +46,7 @@ IQN::Optimize(SeparableFunctionType& functionIn,
   typedef typename MatType::elem_type ElemType;
   typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
   typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+  typedef typename ForwardType<MatType>::bmat ProxyMatType;
 
   typedef Function<SeparableFunctionType, BaseMatType, BaseGradType>
       FullFunctionType;
@@ -81,8 +82,8 @@ IQN::Optimize(SeparableFunctionType& functionIn,
       iterate.n_cols));
   std::vector<BaseMatType> Q(numBatches, BaseMatType(iterate.n_elem,
       iterate.n_elem));
-  BaseMatType initialIterate = arma::randn<arma::Mat<ElemType>>(iterate.n_rows,
-      iterate.n_cols);
+  BaseMatType initialIterate = ProxyMatType(iterate.n_rows, iterate.n_cols,
+      GetFillType<MatType>::randn);
   BaseGradType B(iterate.n_elem, iterate.n_elem);
   B.eye();
 
@@ -103,7 +104,7 @@ IQN::Optimize(SeparableFunctionType& functionIn,
 
     Q[f].eye();
     g += y[f];
-    y[f] /= (double) effectiveBatchSize;
+    y[f] /= (ElemType) effectiveBatchSize;
 
     i += effectiveBatchSize;
   }
@@ -112,8 +113,11 @@ IQN::Optimize(SeparableFunctionType& functionIn,
   BaseGradType gradient(iterate.n_rows, iterate.n_cols);
   BaseMatType u = t[0];
 
+  const size_t actualMaxIterations = (maxIterations == 0) ?
+      std::numeric_limits<size_t>::max() : maxIterations;
+
   Callback::BeginOptimization(*this, function, iterate, callbacks...);
-  for (size_t i = 1; i != maxIterations && !terminate; ++i)
+  for (size_t i = 0; i < actualMaxIterations && !terminate; ++i)
   {
     for (size_t j = 0, f = 0; f < numFunctions; j++)
     {
@@ -124,7 +128,7 @@ IQN::Optimize(SeparableFunctionType& functionIn,
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions -
           it * batchSize);
 
-      if (arma::norm(iterate - t[it]) > 0)
+      if (norm(iterate - t[it]) > 0)
       {
         function.Gradient(iterate, it * batchSize, gradient,
             effectiveBatchSize);
@@ -133,31 +137,34 @@ IQN::Optimize(SeparableFunctionType& functionIn,
         terminate |= Callback::Gradient(*this, function, iterate, gradient,
             callbacks...);
 
-        const BaseMatType s = arma::vectorise(iterate - t[it]);
-        const BaseGradType yy = arma::vectorise(gradient - y[it]);
+        const BaseMatType s = vectorise(iterate - t[it]);
+        const BaseGradType yy = vectorise(gradient - y[it]);
 
         const BaseGradType stochasticHessian = Q[it] + yy * yy.t() /
-            arma::as_scalar(yy.t() * s) - Q[it] * s * s.t() *
-            Q[it] / arma::as_scalar(s.t() * Q[it] * s);
+            as_scalar(yy.t() * s) - Q[it] * s * s.t() *
+            Q[it] / as_scalar(s.t() * Q[it] * s);
+
+        const ElemType negBatches = 1 / ElemType(numBatches);
 
         // Update aggregate Hessian approximation.
-        B += (1.0 / numBatches) * (stochasticHessian - Q[it]);
+        B += negBatches * (stochasticHessian - Q[it]);
 
         // Update aggregate Hessian-variable product.
-        u += arma::reshape((1.0 / numBatches) * (stochasticHessian *
-            arma::vectorise(iterate) - Q[it] * arma::vectorise(t[it])),
-            u.n_rows, u.n_cols);;
+        u += reshape(negBatches * (stochasticHessian *
+            vectorise(iterate) - Q[it] * vectorise(t[it])),
+            u.n_rows, u.n_cols);
 
         // Update aggregate gradient.
-        g += (1.0 / numBatches) * (gradient - y[it]);
+        g += negBatches * (gradient - y[it]);
 
         // Update the function information tables.
         Q[it] = std::move(stochasticHessian);
         y[it] = std::move(gradient);
         t[it] = iterate;
 
-        iterate = arma::reshape(stepSize * B.i() * (u.t() - arma::vectorise(g)),
-            iterate.n_rows, iterate.n_cols) + (1 - stepSize) * iterate;
+        iterate = reshape(ElemType(stepSize) * pinv(B) * (u.t() - vectorise(g)),
+            iterate.n_rows, iterate.n_cols) +
+            (1 - ElemType(stepSize)) * iterate;
 
         terminate |= Callback::StepTaken(*this, function, iterate,
             callbacks...);
@@ -202,8 +209,11 @@ IQN::Optimize(SeparableFunctionType& functionIn,
     }
   }
 
-  Info << "IQN: maximum iterations (" << maxIterations << ") reached; "
-      << "terminating optimization." << std::endl;
+  if (!terminate)
+  {
+    Info << "IQN: maximum iterations (" << maxIterations << ") reached; "
+        << "terminating optimization." << std::endl;
+  }
 
   Callback::EndOptimization(*this, function, iterate, callbacks...);
   return overallObjective;

include/ensmallen_bits/katyusha/katyusha.hpp

@@ -93,7 +93,7 @@ class KatyushaType
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(SeparableFunctionType& function,
            MatType& iterate,

include/ensmallen_bits/katyusha/katyusha_impl.hpp

@@ -45,8 +45,8 @@ template<typename SeparableFunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 KatyushaType<Proximal>::Optimize(
     SeparableFunctionType& function,
     MatType& iterateIn,
@@ -80,20 +80,20 @@ KatyushaType<Proximal>::Optimize(
   if (numFunctions % batchSize != 0)
     ++numBatches; // Capture last few.
 
-  const double tau1 = std::min(0.5,
-      std::sqrt(batchSize * convexity / (3.0 * lipschitz)));
-  const double tau2 = 0.5;
-  const double alpha = 1.0 / (3.0 * tau1 * lipschitz);
-  const double r = 1.0 + std::min(alpha * convexity, 1.0 /
-      (4.0 / innerIterations));
+  const ElemType tau1 = ElemType(std::min(0.5,
+      std::sqrt(batchSize * convexity / (3 * lipschitz))));
+  const ElemType tau2 = ElemType(0.5);
+  const ElemType alpha = 1 / (3 * tau1 * ElemType(lipschitz));
+  const ElemType r = 1 + std::min(alpha * ElemType(convexity),
+                                  ElemType(innerIterations) / 4);
 
   // sum_{j=0}^{m-1} 1 + std::min(alpha * convexity, 1 / (4 * m)^j).
-  double normalizer = 1;
+  ElemType normalizer = 1;
   for (size_t i = 0; i < numBatches; i++)
   {
-    normalizer = r * (normalizer + 1.0);
+    normalizer = r * (normalizer + 1);
   }
-  normalizer = 1.0 / normalizer;
+  normalizer = 1 / normalizer;
 
   // To keep track of where we are and how things are going.
   ElemType overallObjective = 0;
@@ -168,10 +168,10 @@ KatyushaType<Proximal>::Optimize(
 
       f += effectiveBatchSize;
     }
-    fullGradient /= (double) numFunctions;
+    fullGradient /= (ElemType) numFunctions;
 
     // To keep track of where we are and how things are going.
-    double cw = 1;
+    ElemType cw = 1;
     w.zeros();
 
     for (size_t f = 0, currentFunction = 0; (f < innerIterations) && !terminate;
@@ -208,7 +208,7 @@ KatyushaType<Proximal>::Optimize(
       // By the minimality definition of z_{k + 1}, we have that:
       // z_{k+1} − z_k + \alpha * \sigma_{k+1} + \alpha g = 0.
       BaseMatType zNew = z - alpha * (fullGradient + (gradient - gradient0) /
-          (double) batchSize);
+          (ElemType) batchSize);
 
       // Proximal update, choose between Option I and Option II. Shift relative
       // to the Lipschitz constant or take a constant step using the given step
@@ -221,7 +221,7 @@ KatyushaType<Proximal>::Optimize(
         // yk = x0 − 1 / (3L) * \delta3 - ((1 - tau) / (3L)) + tau * alpha)
         // * \delta2 - ((1-tau)^2 / (3L) + (1 - (1 - tau)^2) * alpha) * \delta1,
         // k = 3.
-        y = iterate + 1.0 / (3.0 * lipschitz) * w;
+        y = iterate + 1 / (3 * ElemType(lipschitz)) * w;
       }
       else
       {

include/ensmallen_bits/lbfgs/lbfgs.hpp

@@ -80,7 +80,7 @@ class L_BFGS
            typename MatType,
            typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(FunctionType& function,
            MatType& iterate,
@@ -177,10 +177,11 @@ class L_BFGS
    * @param y Differences between the gradient and the old gradient matrix.
    */
   template<typename MatType, typename CubeType>
-  double ChooseScalingFactor(const size_t iterationNum,
-                             const MatType& gradient,
-                             const CubeType& s,
-                             const CubeType& y);
+  typename MatType::elem_type ChooseScalingFactor(
+      const size_t iterationNum,
+      const MatType& gradient,
+      const CubeType& s,
+      const CubeType& y);
 
   /**
    * Perform a back-tracking line search along the search direction to
@@ -208,7 +209,7 @@ class L_BFGS
                   GradType& gradient,
                   MatType& newIterateTmp,
                   const GradType& searchDirection,
-                  double& finalStepSize,
+                  ElemType& finalStepSize,
                   CallbackTypes&... callbacks);
 
   /**
@@ -224,7 +225,7 @@ class L_BFGS
   template<typename MatType, typename CubeType>
   void SearchDirection(const MatType& gradient,
                        const size_t iterationNum,
-                       const double scalingFactor,
+                       const typename MatType::elem_type scalingFactor,
                        const CubeType& s,
                        const CubeType& y,
                        MatType& searchDirection);

include/ensmallen_bits/lbfgs/lbfgs_impl.hpp

@@ -72,34 +72,48 @@ inline L_BFGS::L_BFGS(const size_t numBasis,
  * @param y Differences between the gradient and the old gradient matrix.
  */
 template<typename MatType, typename CubeType>
-double L_BFGS::ChooseScalingFactor(const size_t iterationNum,
-                                   const MatType& gradient,
-                                   const CubeType& s,
-                                   const CubeType& y)
+typename MatType::elem_type L_BFGS::ChooseScalingFactor(
+    const size_t iterationNum,
+    const MatType& gradient,
+    const CubeType& s,
+    const CubeType& y)
 {
-  typedef typename CubeType::elem_type CubeElemType;
+  typedef typename CubeType::elem_type ElemType;
+  typedef typename ForwardType<CubeType>::bmat BaseMatType;
 
-  constexpr const CubeElemType tol =
-      100 * std::numeric_limits<CubeElemType>::epsilon();
+  constexpr const ElemType tol =
+      100 * std::numeric_limits<ElemType>::epsilon();
 
-  double scalingFactor;
+  ElemType scalingFactor;
   if (iterationNum > 0)
   {
     int previousPos = (iterationNum - 1) % numBasis;
     // Get s and y matrices once instead of multiple times.
-    const arma::Mat<CubeElemType>& sMat = s.slice(previousPos);
-    const arma::Mat<CubeElemType>& yMat = y.slice(previousPos);
+    const BaseMatType& sMat = s.slice(previousPos);
+    const BaseMatType& yMat = y.slice(previousPos);
 
-    const CubeElemType tmp   = arma::dot(yMat, yMat);
-    const CubeElemType denom = (tmp >= tol) ? tmp : CubeElemType(1);
+    const ElemType tmp = dot(yMat, yMat);
+    const ElemType denom = (tmp >= tol) ? tmp : ElemType(1);
+    if (std::isinf(tmp))
+    {
+      Warn << "L-BFGS: squared 2-norm of gradient difference is infinite; "
+          << "try using a higher-precision element type or setting MaxStep() "
+          << "to a smaller value." << std::endl;
+    }
 
-    scalingFactor = arma::dot(sMat, yMat) / denom;
+    scalingFactor = dot(sMat, yMat) / denom;
   }
   else
   {
-    const CubeElemType tmp = arma::norm(gradient, "fro");
+    const ElemType tmp = norm(gradient, "fro");
+    if (std::isinf(tmp))
+    {
+      Warn << "L-BFGS: Frobenius norm of gradient difference is infinite; "
+          << "try using a higher-precision element type or an initial point "
+          << "with a smaller gradient value." << std::endl;
+    }
 
-    scalingFactor = (tmp >= tol) ? (1.0 / tmp) : 1.0;
+    scalingFactor = (tmp >= tol) ? (1 / tmp) : 1;
   }
 
   return scalingFactor;
@@ -118,37 +132,38 @@ double L_BFGS::ChooseScalingFactor(const size_t iterationNum,
 template<typename MatType, typename CubeType>
 void L_BFGS::SearchDirection(const MatType& gradient,
                              const size_t iterationNum,
-                             const double scalingFactor,
+                             const typename MatType::elem_type scalingFactor,
                              const CubeType& s,
                              const CubeType& y,
                              MatType& searchDirection)
 {
+  typedef typename CubeType::elem_type ElemType;
+  typedef typename ForwardType<CubeType>::bmat BaseMatType;
+  typedef typename ForwardType<CubeType>::bcol BaseColType;
+
   // Start from this point.
   searchDirection = gradient;
 
   // See "A Recursive Formula to Compute H * g" in "Updating quasi-Newton
   // matrices with limited storage" (Nocedal, 1980).
-  typedef typename CubeType::elem_type CubeElemType;
 
   // Temporary variables.
-  arma::Col<CubeElemType> rho(numBasis);
-  arma::Col<CubeElemType> alpha(numBasis);
+  BaseColType rho(numBasis);
+  BaseColType alpha(numBasis);
 
   size_t limit = (numBasis > iterationNum) ? 0 : (iterationNum - numBasis);
   for (size_t i = iterationNum; i != limit; i--)
   {
     int translatedPosition = (i + (numBasis - 1)) % numBasis;
+    const BaseMatType& sMat = s.slice(translatedPosition);
+    const BaseMatType& yMat = y.slice(translatedPosition);
 
-    const arma::Mat<CubeElemType>& sMat = s.slice(translatedPosition);
-    const arma::Mat<CubeElemType>& yMat = y.slice(translatedPosition);
+    const ElemType tmp = dot(yMat, sMat);
 
-    const CubeElemType tmp = arma::dot(yMat, sMat);
-
-    rho[iterationNum - i] = (tmp != CubeElemType(0)) ? (1.0 / tmp) :
-        CubeElemType(1);
+    rho[iterationNum - i] = (tmp != ElemType(0)) ? (1 / tmp) : 1;
 
     alpha[iterationNum - i] = rho[iterationNum - i] *
-        arma::dot(sMat, searchDirection);
+        dot(sMat, searchDirection);
 
     searchDirection -= alpha[iterationNum - i] * yMat;
   }
@@ -158,8 +173,8 @@ void L_BFGS::SearchDirection(const MatType& gradient,
   for (size_t i = limit; i < iterationNum; i++)
   {
     int translatedPosition = i % numBasis;
-    double beta = rho[iterationNum - i - 1] *
-        arma::dot(y.slice(translatedPosition), searchDirection);
+    ElemType beta = rho[iterationNum - i - 1] *
+        dot(y.slice(translatedPosition), searchDirection);
     searchDirection += (alpha[iterationNum - i - 1] - beta) *
         s.slice(translatedPosition);
   }
@@ -222,23 +237,27 @@ bool L_BFGS::LineSearch(FunctionType& function,
                         GradType& gradient,
                         MatType& newIterateTmp,
                         const GradType& searchDirection,
-                        double& finalStepSize,
+                        ElemType& finalStepSize,
                         CallbackTypes&... callbacks)
 {
   // Default first step size of 1.0.
-  double stepSize = 1.0;
-  finalStepSize = 0.0; // Set only when we take the step.
+  ElemType stepSize = 1;
+  if (stepSize > ElemType(maxStep))
+    stepSize = ElemType(maxStep);
+  if (stepSize < ElemType(minStep))
+    stepSize = ElemType(minStep);
+  finalStepSize = 0; // Set only when we take the step.
 
   // The initial linear term approximation in the direction of the
   // search direction.
   ElemType initialSearchDirectionDotGradient =
-      arma::dot(gradient, searchDirection);
+      dot(gradient, searchDirection);
 
   // If it is not a descent direction, just report failure.
-  if ( (initialSearchDirectionDotGradient > 0.0)
+  if ( (initialSearchDirectionDotGradient > 0)
     || (std::isfinite(initialSearchDirectionDotGradient) == false) )
   {
-    Warn << "L-BFGS line search direction is not a descent direction "
+    Warn << "L-BFGS: line search direction is not a descent direction "
         << "(terminating)!" << std::endl;
     return false;
   }
@@ -247,17 +266,17 @@ bool L_BFGS::LineSearch(FunctionType& function,
   ElemType initialFunctionValue = functionValue;
 
   // Unit linear approximation to the decrease in function value.
-  ElemType linearApproxFunctionValueDecrease = armijoConstant *
+  ElemType linearApproxFunctionValueDecrease = ElemType(armijoConstant) *
       initialSearchDirectionDotGradient;
 
   // The number of iteration in the search.
   size_t numIterations = 0;
 
   // Armijo step size scaling factor for increase and decrease.
-  const double inc = 2.1;
-  const double dec = 0.5;
-  double width = 0;
-  double bestStepSize = 1.0;
+  const ElemType inc = ElemType(2.1);
+  const ElemType dec = ElemType(0.5);
+  ElemType width = 0;
+  ElemType bestStepSize = 1;
   ElemType bestObjective = std::numeric_limits<ElemType>::max();
 
   while (true)
@@ -270,7 +289,7 @@ bool L_BFGS::LineSearch(FunctionType& function,
 
     if (std::isnan(functionValue))
     {
-      Warn << "L-BFGS objective value is NaN (terminating)!" << std::endl;
+      Warn << "L-BFGS: objective value is NaN (terminating)!" << std::endl;
       return false;
     }
 
@@ -292,7 +311,7 @@ bool L_BFGS::LineSearch(FunctionType& function,
     else
     {
       // Check Wolfe's condition.
-      ElemType searchDirectionDotGradient = arma::dot(gradient,
+      ElemType searchDirectionDotGradient = dot(gradient,
           searchDirection);
 
       if (searchDirectionDotGradient < wolfe *
@@ -346,8 +365,8 @@ template<typename FunctionType,
          typename MatType,
          typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
-typename MatType::elem_type>::type
+typename std::enable_if<IsMatrixType<GradType>::value,
+    typename MatType::elem_type>::type
 L_BFGS::Optimize(FunctionType& function,
                  MatType& iterateIn,
                  CallbackTypes&&... callbacks)
@@ -376,8 +395,10 @@ L_BFGS::Optimize(FunctionType& function,
   const size_t cols = iterate.n_cols;
 
   BaseMatType newIterateTmp(rows, cols);
-  arma::Cube<ElemType> s(rows, cols, numBasis);
-  arma::Cube<ElemType> y(rows, cols, numBasis);
+
+  typedef typename ForwardType<MatType>::bcube BaseCubeType;
+  BaseCubeType s(rows, cols, numBasis);
+  BaseCubeType y(rows, cols, numBasis);
 
   // The old iterate to be saved.
   BaseMatType oldIterate(iterate.n_rows, iterate.n_cols);
@@ -403,6 +424,7 @@ L_BFGS::Optimize(FunctionType& function,
         functionValue, gradient, callbacks...);
 
   ElemType prevFunctionValue;
+  Info << "L-BFGS: initial objective " << functionValue << "." << std::endl;
 
   // The main optimization loop.
   Callback::BeginOptimization(*this, f, iterate, callbacks...);
@@ -417,9 +439,10 @@ L_BFGS::Optimize(FunctionType& function,
     // least one descent step.
     // TODO: to speed this up, investigate use of arma::norm2est() in Armadillo
     // 12.4
-    if (arma::norm(gradient, 2) < minGradientNorm)
+    const ElemType gradNorm = norm(gradient, 2);
+    if (gradNorm < minGradientNorm)
     {
-      Info << "L-BFGS gradient norm too small (terminating successfully)."
+      Info << "L-BFGS: gradient norm too small (terminating successfully)."
           << std::endl;
       break;
     }
@@ -427,24 +450,24 @@ L_BFGS::Optimize(FunctionType& function,
     // Break if the objective is not a number.
     if (std::isnan(functionValue))
     {
-      Warn << "L-BFGS terminated with objective " << functionValue << "; "
+      Warn << "L-BFGS: terminated with objective " << functionValue << "; "
           << "are the objective and gradient functions implemented correctly?"
           << std::endl;
       break;
     }
 
     // Choose the scaling factor.
-    double scalingFactor = ChooseScalingFactor(itNum, gradient, s, y);
-    if (scalingFactor == 0.0)
+    ElemType scalingFactor = ChooseScalingFactor(itNum, gradient, s, y);
+    if (scalingFactor == 0)
     {
-      Info << "L-BFGS scaling factor computed as 0 (terminating successfully)."
+      Info << "L-BFGS: scaling factor computed as 0 (terminating successfully)."
           << std::endl;
       break;
     }
 
     if (std::isfinite(scalingFactor) == false)
       {
-      Warn << "L-BFGS scaling factor is not finite.  Stopping optimization."
+      Warn << "L-BFGS: scaling factor is not finite.  Stopping optimization."
            << std::endl;
       break;
       }
@@ -457,31 +480,34 @@ L_BFGS::Optimize(FunctionType& function,
     oldIterate = iterate;
     oldGradient = gradient;
 
-    double stepSize; // Set by LineSearch().
+    ElemType stepSize; // Set by LineSearch().
     if (!LineSearch(f, functionValue, iterate, gradient, newIterateTmp,
         searchDirection, stepSize, callbacks...))
     {
-      Warn << "Line search failed.  Stopping optimization." << std::endl;
+      Warn << "L-BFGS: line search failed.  Stopping optimization."
+          << std::endl;
       break; // The line search failed; nothing else to try.
     }
 
     // It is possible that the difference between the two coordinates is zero.
     // In this case we terminate successfully.
-    if (stepSize == 0.0)
+    if (stepSize == 0)
     {
-      Info << "L-BFGS step size of 0 (terminating successfully)."
+      Info << "L-BFGS: computed step size of 0 (terminating successfully)."
           << std::endl;
       break;
     }
 
+    Info << "L-BFGS: iteration " << itNum << ", objective " << functionValue
+        << ", step size " << stepSize << "." << std::endl;
+
     // If we can't make progress on the gradient, then we'll also accept
     // a stable function value.
-    const double denom = std::max(
-        std::max(std::abs(prevFunctionValue), std::abs(functionValue)),
-        (ElemType) 1.0);
+    const ElemType denom = std::max(ElemType(1),
+        std::max(std::abs(prevFunctionValue), std::abs(functionValue)));
     if ((prevFunctionValue - functionValue) / denom <= factr)
     {
-      Info << "L-BFGS function value stable (terminating successfully)."
+      Info << "L-BFGS: function value stable (terminating successfully)."
           << std::endl;
       break;
     }
@@ -499,4 +525,3 @@ L_BFGS::Optimize(FunctionType& function,
 } // namespace ens
 
 #endif // ENSMALLEN_LBFGS_LBFGS_IMPL_HPP
-