Update and release version 2.14.2.

Fix implementation of fonesca fleming problem function f1 and f2     type usage and negative signs.

.appveyor.yml

@@ -0,0 +1,57 @@
+environment:
+  ARMADILLO_DOWNLOAD: "http://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz"
+  BLAS_LIBRARY: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a"
+  BLAS_LIBRARY_DLL: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll"
+
+  matrix:
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+      VSVER: Visual Studio 14 2015 Win64
+      MSBUILD: C:\Program Files (x86)\MSBuild\14.0\bin\MSBuild.exe
+
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      VSVER: Visual Studio 15 2017 Win64
+      MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe
+
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+      VSVER: Visual Studio 16 2019
+      MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe
+
+configuration: Release
+
+install:
+  - ps: nuget install OpenBLAS -o "${env:APPVEYOR_BUILD_FOLDER}"
+
+build_script:
+  # First, download and build Armadillo.
+  - cd ..
+  - appveyor DownloadFile %ARMADILLO_DOWNLOAD% -FileName armadillo.tar.xz
+  - 7z x armadillo.tar.xz -so -txz | 7z x -si -ttar > nul
+  - cd armadillo-8.400.0 && mkdir build && cd build
+  - >
+    cmake -G "%VSVER%"
+    -DBLAS_LIBRARY:FILEPATH=%BLAS_LIBRARY%
+    -DLAPACK_LIBRARY:FILEPATH=%BLAS_LIBRARY%
+    -DCMAKE_PREFIX:FILEPATH="%APPVEYOR_BUILD_FOLDER%/armadillo"
+    -DBUILD_SHARED_LIBS=OFF
+    -DCMAKE_BUILD_TYPE=Release ..
+  - >
+    "%MSBUILD%" "armadillo.sln"
+    /m /verbosity:quiet /p:Configuration=Release;Platform=x64
+  - cd ../..
+
+  # Now build ensmallen.
+  - cd ensmallen && mkdir build && cd build
+  - >
+    cmake -G "%VSVER%"
+    -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-8.400.0/include/
+    -DARMADILLO_LIBRARIES=%BLAS_LIBRARY%
+    -DLAPACK_LIBRARY=%BLAS_LIBRARY%
+    -DBLAS_LIBRARY=%BLAS_LIBRARY%
+    -DCMAKE_BUILD_TYPE=Release ..
+  - >
+    "%MSBUILD%" "ensmallen.sln"
+    /m /verbosity:minimal /nologo /p:BuildInParallel=true
+
+  # Run tests after copying libraries.
+  - ps: cp C:\projects\ensmallen\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* C:\projects\ensmallen\build\
+  - ctest -C Release -V --output-on-failure .

.github/ISSUE_TEMPLATE/0-bug.md

@@ -0,0 +1,40 @@
+---
+name: Bug report
+about: Use this template for reporting a bug that you have found in ensmallen.
+title: ''
+labels: 't: bug report, s: unanswered'
+assignees: ''
+
+---
+
+<!--
+
+Welcome!  Please fill out the template below; that makes it easier for us to 
+quickly figure out what the issue is and solve it.  Thanks!
+
+-->
+
+#### Issue description
+
+<!-- Describe your issue here. -->
+
+#### Your environment
+
+ * version of ensmallen:
+ * operating system:
+ * compiler:
+ * version of Armadillo:
+ * any other environment information you think is relevant:
+
+#### Steps to reproduce
+
+<!-- Tell us how to reproduce the issue; please provide a working example if
+possible! -->
+
+#### Expected behavior
+
+<!-- Tell us what should happen. -->
+
+#### Actual behavior
+
+<!-- Tell us what happened instead. -->

.github/ISSUE_TEMPLATE/1-documentation.md

@@ -0,0 +1,26 @@
+---
+name: Documentation issue
+about: Use this template to report an issue you've found with the documentation.
+title: ''
+labels: 't: bug report, c: documentation, s: unanswered'
+assignees: ''
+
+---
+
+<!--
+
+Welcome!  Unfortunately not all documentation is perfect, and if you're opening
+a documentation issue we are interested in fixing it.  Please fill out the
+template below so that we can solve the problem more quickly; or, alternately,
+open a PR with a fix, if you like.
+
+-->
+
+#### Problem location
+
+<!-- Link to incorrect website or location of source file with bad
+documentation. -->
+
+#### Description of problem
+
+<!-- Tell us what is wrong with the documentation so we can fix it. -->

.github/ISSUE_TEMPLATE/2-question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: Use this template for other problems, requests, or questions.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+<!--
+
+Welcome!  If you have a question you'd like to ask, you can do it here or on the
+mlpack mailing list; see also http://mlpack.org/help.html.
+
+If you're looking for how to get involved and contribute, there's no need to
+open an issue---you can see http://www.mlpack.org/involved.html instead.
+
+-->

.gitignore

@@ -0,0 +1 @@
+build/

.travis.yml

@@ -1,26 +1,32 @@
-sudo: required
+os: linux
 dist: trusty
 language: cpp
 
 env:
-  - ARMADILLO=latest
+  - ARMADILLO=latest SANITY_HISTORY=perform
   - ARMADILLO=minimum
 
-before_install:
-  - sudo apt-get update
-  - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++
-  - if [ $ARMADILLO == "latest" ]; then
-      curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.*<a href="\(armadillo-[0-9]*.[0-9]*.[0-9]*.tar.xz\)".*$/\1/' | tail -1` | tar xvJ && cd armadillo*;
-    fi
-  - if [ $ARMADILLO == "minimum" ]; then
-      curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-6.500.5.tar.gz | tar xvz && cd armadillo*;
-    fi
-  - cmake . && make && sudo make install && cd ..
+stages:
+  - test
+  - name: sanity
+    if: type = pull_request AND env(SANITY_HISTORY) = "perform"
 
-install:
-  - mkdir build && cd build && cmake .. && make -j2
+jobs:
+  include:
+    - stage: sanity
+      name: "HISTORY.md Check"
+      script: sh ./scripts/history-update-check.sh
 
 script:
+  - sudo apt-get update
+  - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ xz-utils
+  - if [ $ARMADILLO == "latest" ]; then
+      curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.*<a href="\(armadillo-[0-9]*.[0-9]*.[0-9]*.tar.xz\)".*$/\1/' | tail -1` | tar xvJ && cd armadillo*;
+    else
+      curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz | tar -xvJ && cd armadillo*;
+    fi
+  - cmake . && make && sudo make install && cd ..
+  - mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" .. && make -j2
   - CTEST_OUTPUT_ON_FAILURE=1 travis_wait 30 ctest -j2
 
 notifications:

CMake/FindArmadillo.cmake

@@ -366,4 +366,10 @@ mark_as_advanced(
   ARMADILLO_INCLUDE_DIR
   ARMADILLO_LIBRARIES)
 
+if (ARMADILLO_FOUND AND NOT TARGET Armadillo:Armadillo)
+  add_library(Armadillo::Armadillo INTERFACE IMPORTED)
+  set_target_properties(Armadillo::Armadillo PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ARMADILLO_INCLUDE_DIR}"
+      INTERFACE_LINK_LIBRARIES "${ARMADILLO_LIBRARIES}")
+endif()
+
 #======================

CMake/ensmallen-config.cmake.in

@@ -0,0 +1,4 @@
+@PACKAGE_INIT@
+
+include(${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake)
+check_required_components(ensmallen)

CMakeLists.txt

@@ -1,55 +1,92 @@
 # ensmallen CMake configuration.  This project has no configurable options---it
 # just installs the headers to the install location, and optionally builds the
 # test program.
-cmake_minimum_required(VERSION 2.8.10)
-project(ensmallen C CXX)
+cmake_minimum_required(VERSION 3.3.2)
+project(ensmallen 
+        LANGUAGES C CXX)
 
+# Configurable options for CMake.
 option(USE_OPENMP "If available, use OpenMP for parallelization." ON)
+option(BUILD_TESTS "Build tests." ON)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake")
 
-# Ensure we have C++11 features.  Since we support CMake < 3.1, this needs a
-# little bit of special handling.
-if ((${CMAKE_MAJOR_VERSION} LESS 3 OR
-    (${CMAKE_MAJOR_VERSION} EQUAL 3 AND ${CMAKE_MINOR_VERSION} LESS 1))
-    AND NOT FORCE_CXX11)
-  # Older versions of CMake do not support target_compile_features(), so we have
-  # to use something kind of hacky.
-  include(CMake/CXX11.cmake)
-  check_for_cxx11_compiler(HAS_CXX11)
-  if(NOT HAS_CXX11)
-    message(FATAL_ERROR "No C++11 compiler available!")
-  endif()
-  enable_cxx11()
+# Set required C++ standard to C++11.
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Extract version from sources.
+set(ENSMALLEN_VERSION_FILE_NAME "${PROJECT_SOURCE_DIR}/include/ensmallen_bits/ens_version.hpp")
+
+if(NOT EXISTS ${ENSMALLEN_VERSION_FILE_NAME})
+  message(FATAL_ERROR "Can't read ${ENSMALLEN_VERSION_FILE_NAME}")
+endif()
+
+file(READ ${ENSMALLEN_VERSION_FILE_NAME} ENSMALLEN_VERSION_FILE_CONTENTS)
+string(REGEX REPLACE ".*#define ENS_VERSION_MAJOR ([0-9]+).*" "\\1" ENSMALLEN_VERSION_MAJOR "${ENSMALLEN_VERSION_FILE_CONTENTS}")
+string(REGEX REPLACE ".*#define ENS_VERSION_MINOR ([0-9]+).*" "\\1" ENSMALLEN_VERSION_MINOR "${ENSMALLEN_VERSION_FILE_CONTENTS}")
+string(REGEX REPLACE ".*#define ENS_VERSION_PATCH ([0-9]+).*" "\\1" ENSMALLEN_VERSION_PATCH "${ENSMALLEN_VERSION_FILE_CONTENTS}")
+
+message(STATUS "Configuring ensmallen ${ENSMALLEN_VERSION_MAJOR}.${ENSMALLEN_VERSION_MINOR}.${ENSMALLEN_VERSION_PATCH}")
+set(VERSION "${ENSMALLEN_VERSION_MAJOR}.${ENSMALLEN_VERSION_MINOR}.${ENSMALLEN_VERSION_PATCH}")
+
+# Create library target.
+add_library(ensmallen INTERFACE)
+target_include_directories(ensmallen INTERFACE
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:include>)
+
+# Set warning flags for target.
+if(MSVC)
+  target_compile_options(ensmallen INTERFACE $<BUILD_INTERFACE:/Wall>)
 else()
-  # set required standard to c++11
-  set(CMAKE_CXX_STANDARD 11)
-  set(CMAKE_CXX_STANDARD_REQUIRED ON)
-endif ()
+  target_compile_options(ensmallen INTERFACE $<BUILD_INTERFACE:-Wall -Wpedantic -Wunused-parameter>)
+endif()
 
-# Detect OpenMP support in a compiler. If the compiler supports OpenMP, flags
-# to compile with OpenMP are returned and added for compilation.
-if (USE_OPENMP)
-  find_package(OpenMP)
-endif ()
+# Find OpenMP and link it.
+if(USE_OPENMP)
+  if(NOT TARGET OpenMP::OpenMP_CXX)
+    find_package(Threads REQUIRED)
+    add_library(OpenMP::OpenMP_CXX IMPORTED INTERFACE)
+    set_property(TARGET OpenMP::OpenMP_CXX
+        PROPERTY INTERFACE_COMPILE_OPTIONS ${OpenMP_CXX_FLAGS})
+    # Only works if the same flag is passed to the linker; use CMake 3.9+ otherwise (Intel, AppleClang).
+    set_property(TARGET OpenMP::OpenMP_CXX
+        PROPERTY INTERFACE_LINK_LIBRARIES ${OpenMP_CXX_FLAGS} Threads::Threads)
+  endif()
+  target_link_libraries(ensmallen INTERFACE OpenMP::OpenMP_CXX)
+endif()
 
-if (OPENMP_FOUND)
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-else ()
-  # Disable warnings for all the unknown OpenMP pragmas.
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas")
-endif ()
+# Find Armadillo and link it.
+find_package(Armadillo 8.400.0 REQUIRED)
+target_link_libraries(ensmallen INTERFACE Armadillo::Armadillo)
 
-# The only dependency we need is Armadillo.
-#
-# We keep the minimum version in sync with mlpack, otherwise we could have
-# irritating compatibility issues.
-find_package(Armadillo 6.500.0 REQUIRED)
-include_directories(BEFORE "${ARMADILLO_INCLUDE_DIR}")
-include_directories(BEFORE "${CMAKE_SOURCE_DIR}/include/")
+# Set helper variables for creating the version, config and target files.
+include(CMakePackageConfigHelpers)
+set(ENSMALLEN_CMAKE_DIR "lib/cmake/ensmallen" CACHE STRING
+      "Installation directory for cmake files, relative to ${CMAKE_INSTALL_PREFIX}.")
+set(VERSION_CONFIG "${PROJECT_BINARY_DIR}/ensmallen-config-version.cmake")
+set(PROJECT_CONFIG "${PROJECT_BINARY_DIR}/ensmallen-config.cmake")
+set(TARGETS_EXPORT_NAME ensmallen-targets)
 
-# Install the headers to the correct location.
+# Generate the version, config and target files into the build directory.
+write_basic_package_version_file(${VERSION_CONFIG}
+    VERSION ${VERSION}
+    COMPATIBILITY AnyNewerVersion)
+configure_package_config_file(${PROJECT_SOURCE_DIR}/CMake/ensmallen-config.cmake.in
+    ${PROJECT_CONFIG}
+    INSTALL_DESTINATION ${ENSMALLEN_CMAKE_DIR})
+export(TARGETS ensmallen NAMESPACE ensmallen::
+    FILE ${PROJECT_BINARY_DIR}/${TARGETS_EXPORT_NAME}.cmake)
+
+# Install version, config and target files.
+install(FILES ${PROJECT_CONFIG} ${VERSION_CONFIG}
+    DESTINATION ${ENSMALLEN_CMAKE_DIR})
+install(EXPORT ${TARGETS_EXPORT_NAME} DESTINATION ${ENSMALLEN_CMAKE_DIR}
+    NAMESPACE ensmallen::)
+
+# Export the targets and install the header files.
+install(TARGETS ensmallen EXPORT ${TARGETS_EXPORT_NAME} DESTINATION lib)
 install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/ensmallen_bits"
         DESTINATION "${CMAKE_INSTALL_PREFIX}/include"
         PATTERN "*~" EXCLUDE
@@ -57,6 +94,8 @@ install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/ensmallen_bits"
 install(FILES ${CMAKE_SOURCE_DIR}/include/ensmallen.hpp
         DESTINATION "${CMAKE_INSTALL_PREFIX}/include")
 
+# Enable testing and build tests.
 enable_testing()
-
-add_subdirectory(tests)
+if (BUILD_TESTS)
+  add_subdirectory(tests)
+endif()

CONTRIBUTING.md

@@ -42,4 +42,140 @@ you have written a test like this, make sure it does not fail often by
 uncommenting the code that sets a random seed in `tests/main.cpp` and running
 your test many times.
 
-Information on how to build and run the tests is in the main README.md file.
+### Install Dependencies
+
+<details open>
+<summary>Linux</summary>
+
+Use your distributions' package manager to install the required dependencies. For Ubuntu/Debian, the commands are as shown below.
+
+```bash
+$ sudo apt-get update
+$ sudo apt-get install libarmadillo-dev cmake
+```
+
+To install the dependencies on Fedora/RHEL/CentOS, you can use:
+
+```bash
+$ yum install armadillo-devel cmake
+```
+
+You can install **ensmallen** on Arch from AUR using:
+
+```bash
+$ git clone https://aur.archlinux.org/ensmallen.git
+$ cd ensmallen
+$ makepkg -si
+```
+
+</details>
+
+<details open>
+<summary>macOS</summary>
+
+You can install **ensmallen** directly with [Homebrew](https://brew.sh).
+
+```bash
+$ brew install ensmallen
+```
+
+If you want to build **ensmallen** tests from source, use a package manager like [Homebrew](https://brew.sh) to get the necessary dependencies.
+
+```bash
+$ brew install armadillo cmake
+```
+</details>
+
+<details open>
+<summary>Windows</summary>
+
+You can install **ensmallen** directly by using [vcpkg](https://github.com/microsoft/vcpkg).
+
+```
+vcpkg install ensmallen:x64-windows
+```
+</details>
+
+### Build and Test
+
+This section describes how to build the **ensmallen** tests from source. **ensmallen** uses CMake as its build system and [Catch2](https://github.com/catchorg/Catch2) as the unit test framework.
+
+First, clone the source code from Github and change into the cloned directory. Or alternatively, you can download the latest relese from the [website](http://ensmallen.org) and extract it.
+
+```bash
+$ git clone https://github.com/mlpack/ensmallen
+$ cd ensmallen
+
+# - or -
+
+$ wget http://ensmallen.org/files/ensmallen-2.14.2.tar.gz
+$ tar -xvzpf ensmallen-2.14.2.tar.gz
+$ cd ensmallen-latest
+```
+
+Next, make a build directory and change into that directory.
+
+```bash
+$ mkdir build
+$ cd build
+```
+
+Then, run the cmake command followed by the make command in the build directory. If the cmake command fails, you probably have missing dependencies.
+
+```bash
+$ cmake ..
+# or with -w flag to inhibit all warning messages
+# $ cmake -DCMAKE_CXX_FLAGS="-w" -DCMAKE_C_FLAGS="-w" .. #
+$ make
+# or alternately make with the -jN flag to run parallel jobs
+# $ make -j4
+```
+
+Now, you can either run all of the tests or an individual test case with:
+
+```bash
+$ ./ensmallen_tests
+$ ./ensmallen_tests <test name>
+```
+
+You can list all tests with:
+
+```bash
+$ ./ensmallen-tests -l
+
+ensmallen version: 2.10.5 (Fried Chicken)
+armadillo version: 9.800.3 (Horizon Scraper)
+All available test cases:
+  SimpleAdaDeltaTestFunction
+      [AdaDeltaTest]
+...
+262 test cases
+```
+
+
+## Release process
+
+New releases of ensmallen can be performed almost-automatically with the
+`rel/ensmallen-release.sh` script.  Releases can only be performed by
+contributors with push permissions to the repository.  Before making a release,
+make sure that all the tests are passing and the release number satisfies the
+versioning guidelines in `UPDATING.md` and make sure that `HISTORY.md` is
+up-to-date with the new release's release notes (and date).
+
+The script should be run, e.g.,
+
+```
+$ rel/ensmallen-release.sh 2 10 0 "Fried Chicken"
+```
+
+If the version is a new minor version (or major version), choose a name for the
+release.  Previous release names have generally been entirely arbitrary.
+
+Then, after running the script, a release needs to be done on the Github
+website:
+
+https://github.com/mlpack/ensmallen/releases/new
+
+The format for the release notes is just the release date at the top (e.g.,
+`Released Sept. 7th, 2019`), followed by the Markdown-formatted HISTORY.md
+updates for that release.

COPYRIGHT.txt

@@ -35,7 +35,11 @@ Copyright:
   Copyright 2018, Shikhar Jaiswal <jaiswalshikhar87@gmail.com>
   Copyright 2018, Conrad Sanderson
   Copyright 2018, Dan Timson
-  
+  Copyright 2019, Rahul Ganesh Prabhu
+  Copyright 2019, Roberto Hueso <robertohueso96@gmail.com>
+  Copyright 2019, Sayan Goswami <sayan.goswami.106@gmail.com>
+  Copyright 2020, Joe Dinius <josephwdinius@gmail.com>
+
 License: BSD-3-clause
   All rights reserved.
   .

HISTORY.md

@@ -1,51 +1,304 @@
-### ensmallen 1.13.0
+### ensmallen 2.14.2: "No Direction Home"
+###### 2020-08-31
+ * Fix implementation of fonesca fleming problem function f1 and f2 
+   type usage and negative signs. ([#223](https://github.com/mlpack/ensmallen/pull/223))
+
+### ensmallen 2.14.1: "No Direction Home"
+###### 2020-08-19
+ * Fix release script (remove hardcoded information, trim leading whitespaces
+   introduced by `wc -l` in MacOS)
+    ([#216](https://github.com/mlpack/ensmallen/pull/216),
+     [#220](https://github.com/mlpack/ensmallen/pull/220)).
+
+ * Adjust tolerance for AugLagrangian convergence based on element type
+   ([#217](https://github.com/mlpack/ensmallen/pull/217)).
+
+### ensmallen 2.14.0: "No Direction Home"
+###### 2020-08-10
+ * Add NSGA2 optimizer for multi-objective functions
+    ([#149](https://github.com/mlpack/ensmallen/pull/149)).
+
+ * Update automatic website update release script
+   ([#207](https://github.com/mlpack/ensmallen/pull/207)).
+
+ * Clarify and fix documentation for constrained optimizers
+   ([#201](https://github.com/mlpack/ensmallen/pull/201)).
+
+ * Fix L-BFGS convergence when starting from a minimum
+   ([#201](https://github.com/mlpack/ensmallen/pull/201)).
+
+### ensmallen 2.13.0: "Automatically Automated Automation"
+###### 2020-07-15
+ * Fix CMake package export
+    ([#198](https://github.com/mlpack/ensmallen/pull/198)).
+
+ * Allow early stop callback to accept a lambda function
+   ([#165](https://github.com/mlpack/ensmallen/pull/165)).
+
+### ensmallen 2.12.1: "Stir Crazy"
+###### 2020-04-20
+ * Fix total number of epochs and time estimation for ProgressBar callback
+    ([#181](https://github.com/mlpack/ensmallen/pull/181)).
+
+ * Handle SpSubview_col and SpSubview_row in Armadillo 9.870
+    ([#194](https://github.com/mlpack/ensmallen/pull/194)).
+
+ * Minor documentation fixes
+    ([#197](https://github.com/mlpack/ensmallen/pull/197)).
+
+### ensmallen 2.12.0: "Stir Crazy"
+###### 2020-03-28
+ * Correction in the formulation of sigma in CMA-ES
+    ([#183](https://github.com/mlpack/ensmallen/pull/183)).
+
+ * Remove deprecated methods from PrimalDualSolver implementation
+    ([#185](https://github.com/mlpack/ensmallen/pull/185).
+
+ * Update logo ([#186](https://github.com/mlpack/ensmallen/pull/186)).
+
+### ensmallen 2.11.5: "The Poster Session Is Full"
+###### 2020-03-11
+  * Change "mathematical optimization" term to "numerical optimization" in the
+    documentation ([#177](https://github.com/mlpack/ensmallen/pull/177)).
+
+### ensmallen 2.11.4: "The Poster Session Is Full"
+###### 2020-03-03
+  * Require new HISTORY.md entry for each PR.
+    ([#171](https://github.com/mlpack/ensmallen/pull/171),
+     [#172](https://github.com/mlpack/ensmallen/pull/172),
+     [#175](https://github.com/mlpack/ensmallen/pull/175)).
+
+  * Update/fix example documentation
+    ([#174](https://github.com/mlpack/ensmallen/pull/174)).
+
+### ensmallen 2.11.3: "The Poster Session Is Full"
+###### 2020-02-19
+  * Prevent spurious compiler warnings
+    ([#161](https://github.com/mlpack/ensmallen/pull/161)).
+
+  * Fix minor memory leaks
+    ([#167](https://github.com/mlpack/ensmallen/pull/167)).
+
+  * Revamp CMake configuration
+    ([#152](https://github.com/mlpack/ensmallen/pull/152)).
+
+### ensmallen 2.11.2: "The Poster Session Is Full"
+###### 2020-01-16
+  * Allow callback instantiation for SGD based optimizer
+    ([#138](https://github.com/mlpack/ensmallen/pull/155)).
+
+  * Minor test stability fixes on i386
+    ([#156](https://github.com/mlpack/ensmallen/pull/156)).
+
+  * Fix Lookahead MaxIterations() check.
+    ([#159](https://github.com/mlpack/ensmallen/pull/159)).
+
+### ensmallen 2.11.1: "The Poster Session Is Full"
+###### 2019-12-28
+  * Fix Lookahead Synchronization period type
+    ([#153](https://github.com/mlpack/ensmallen/pull/153)).
+
+### ensmallen 2.11.0: "The Poster Session Is Full"
+###### 2019-12-24
+  * Add Lookahead
+    ([#138](https://github.com/mlpack/ensmallen/pull/138)).
+
+  * Add AdaBound and AMSBound
+    ([#137](https://github.com/mlpack/ensmallen/pull/137)).
+
+### ensmallen 2.10.5: "Fried Chicken"
+###### 2019-12-13
+  * SGD callback test 32-bit safety (big number)
+    ([#143](https://github.com/mlpack/ensmallen/pull/143)).
+
+  * Use "arbitrary" and "separable" terms in static function type checks
+    ([#145](https://github.com/mlpack/ensmallen/pull/145)).
+
+  * Remove 'using namespace std' from `problems/` files
+    ([#147](https://github.com/mlpack/ensmallen/pull/147)).
+
+### ensmallen 2.10.4: "Fried Chicken"
+###### 2019-11-18
+  * Add optional tests building.
+    ([#141](https://github.com/mlpack/ensmallen/pull/141)).
+
+  * Make code samples collapsible in the documentation.
+    ([#140](https://github.com/mlpack/ensmallen/pull/140)).
+
+### ensmallen 2.10.3: "Fried Chicken"
+###### 2019-09-26
+  * Fix ParallelSGD runtime bug.
+    ([#135](https://github.com/mlpack/ensmallen/pull/135)).
+
+  * Add additional L-BFGS convergence check
+    ([#136](https://github.com/mlpack/ensmallen/pull/136)).
+
+### ensmallen 2.10.2: "Fried Chicken"
+###### 2019-09-11
+  * Add release script to rel/ for maintainers
+    ([#128](https://github.com/mlpack/ensmallen/pull/128)).
+
+  * Fix Armadillo version check
+    ([#133](https://github.com/mlpack/ensmallen/pull/133)).
+
+### ensmallen 2.10.1: "Fried Chicken"
+###### 2019-09-10
+  * Documentation fix for callbacks
+    ([#129](https://github.com/mlpack/ensmallen/pull/129).
+
+  * Compatibility fixes for ensmallen 1.x
+    ([#131](https://github.com/mlpack/ensmallen/pull/131)).
+
+### ensmallen 2.10.0: "Fried Chicken"
+###### 2019-09-07
+  * All `Optimize()` functions now take any matrix type; so, e.g., `arma::fmat`
+    or `arma::sp_mat` can be used for optimization.  See the documentation for
+    more details ([#113](https://github.com/mlpack/ensmallen/pull/113),
+    [#119](https://github.com/mlpack/ensmallen/pull/119)).
+
+  * Introduce callback support.  Callbacks can be appended as the last arguments
+    of an `Optimize()` call, and can perform custom behavior at different points
+    during the optimization.  See the documentation for more details
+    ([#119](https://github.com/mlpack/ensmallen/pull/119)).
+
+  * Slight speedups for `FrankWolfe` optimizer
+    ([#127](https://github.com/mlpack/ensmallen/pull/127)).
+
+### ensmallen 1.16.2: "Loud Alarm Clock"
+###### 2019-08-12
+  * Fix PSO return type bug
+    ([#126](https://github.com/mlpack/ensmallen/pull/126)).
+
+### ensmallen 1.16.1: "Loud Alarm Clock"
+###### 2019-08-11
+  * Update HISTORY.md to use Markdown links to the PR and add release names.
+
+  * Fix PSO return type bug
+    ([#124](https://github.com/mlpack/ensmallen/pull/124)).
+
+### ensmallen 1.16.0: "Loud Alarm Clock"
+###### 2019-08-09
+  * Add option to avoid computing exact objective at the end of the optimization
+    ([#109](https://github.com/mlpack/ensmallen/pull/109)).
+
+  * Fix handling of curvature for BigBatchSGD
+    ([#118](https://github.com/mlpack/ensmallen/pull/118)).
+
+  * Reduce runtime of tests
+    ([#118](https://github.com/mlpack/ensmallen/pull/118)).
+
+  * Introduce local-best particle swarm optimization, `LBestPSO`, for
+    unconstrained optimization problems
+    ([#86](https://github.com/mlpack/ensmallen/pull/86)).
+
+### ensmallen 1.15.1: "Wrong Side Of The Road"
+###### 2019-05-22
+  * Fix `-Wreorder` in `qhadam` warning
+    ([#115](https://github.com/mlpack/ensmallen/pull/115)).
+
+  * Fix `-Wunused-private-field` warning in `spsa`
+    ([#115](https://github.com/mlpack/ensmallen/pull/115)).
+
+  * Add more warning output for gcc/clang
+    ([#116](https://github.com/mlpack/ensmallen/pull/116)).
+
+### ensmallen 1.15.0: "Wrong Side Of The Road"
+###### 2019-05-14
+  * Added QHAdam and QHSGD optimizers
+    ([#81](https://github.com/mlpack/ensmallen/pull/81)).
+
+### ensmallen 1.14.4: "Difficult Crimp"
+###### 2019-05-12
+  * Fixes for BigBatchSGD ([#91](https://github.com/mlpack/ensmallen/pull/91)).
+
+### ensmallen 1.14.3: "Difficult Crimp"
+###### 2019-05-06
+  * Handle `eig_sym()` failures correctly
+    ([#100](https://github.com/mlpack/ensmallen/pull/100)).
+
+### ensmallen 1.14.2: "Difficult Crimp"
+###### 2019-03-14
+  * SPSA test tolerance fix
+    ([#97](https://github.com/mlpack/ensmallen/pull/97)).
+
+  * Minor documentation fixes (#95, #98).
+
+  * Fix newlines at end of file
+    ([#92](https://github.com/mlpack/ensmallen/pull/92)).
+
+### ensmallen 1.14.1: "Difficult Crimp"
+###### 2019-03-09
+  * Fixes for SPSA ([#87](https://github.com/mlpack/ensmallen/pull/87)).
+
+  * Optimized CNE and DE ([#90](https://github.com/mlpack/ensmallen/pull/90)).
+    Changed initial population generation in CNE to be a normal distribution
+    about the given starting point, which should accelerate convergence.
+
+### ensmallen 1.14.0: "Difficult Crimp"
+###### 2019-02-20
+  * Add DE optimizer ([#77](https://github.com/mlpack/ensmallen/pull/77)).
+
+  * Fix for Cholesky decomposition in CMAES
+    ([#83](https://github.com/mlpack/ensmallen/pull/83)).
+
+### ensmallen 1.13.2: "Coronavirus Invasion"
+###### 2019-02-18
+ * Minor documentation fixes ([#82](https://github.com/mlpack/ensmallen/pull/82)).
+
+### ensmallen 1.13.1: "Coronavirus Invasion"
+###### 2019-01-24
+ * Fix -Wreorder warning ([#75](https://github.com/mlpack/ensmallen/pull/75)).
+
+### ensmallen 1.13.0: "Coronavirus Invasion"
 ###### 2019-01-14
- * Enhance options for AugLagrangian optimizer (#66).
+ * Enhance options for AugLagrangian optimizer
+   ([#66](https://github.com/mlpack/ensmallen/pull/66)).
 
- * Add SPSA optimizer (#69).
+ * Add SPSA optimizer ([#69](https://github.com/mlpack/ensmallen/pull/69)).
 
-### ensmallen 1.12.2
+### ensmallen 1.12.2: "New Year's Party"
 ###### 2019-01-05
  * Fix list of contributors.
 
-### ensmallen 1.12.1
+### ensmallen 1.12.1: "New Year's Party"
 ###### 2019-01-03
  * Make sure all files end with newlines.
 
-### ensmallen 1.12.0
+### ensmallen 1.12.0: "New Year's Party"
 ###### 2018-12-30
  * Add link to ensmallen PDF to README.md.
 
  * Minor documentation fixes.  Remove too-verbose documentation from source for
-   each optimizer (#61).
+   each optimizer ([#61](https://github.com/mlpack/ensmallen/pull/61)).
 
- * Add FTML optimizer (#48).
+ * Add FTML optimizer ([#48](https://github.com/mlpack/ensmallen/pull/48)).
 
- * Add SWATS optimizer (#42).
+ * Add SWATS optimizer ([#42](https://github.com/mlpack/ensmallen/pull/42)).
 
- * Add Padam optimizer (#46).
+ * Add Padam optimizer ([#46](https://github.com/mlpack/ensmallen/pull/46)).
 
- * Add Eve optimizer (#45).
+ * Add Eve optimizer ([#45](https://github.com/mlpack/ensmallen/pull/45)).
 
- * Add ResetPolicy() to SGD-like optimizers (#60).
+ * Add ResetPolicy() to SGD-like optimizers
+   ([#60](https://github.com/mlpack/ensmallen/pull/60)).
 
-### ensmallen 1.11.1
+### ensmallen 1.11.1: "Jet Lag"
 ###### 2018-11-29
  * Minor documentation fixes.
 
-### ensmallen 1.11.0
+### ensmallen 1.11.0: "Jet Lag"
 ###### 2018-11-28
  * Add WNGrad optimizer.
 
  * Fix header name in documentation samples.
 
-### ensmallen 1.10.1
+### ensmallen 1.10.1: "Corporate Catabolism"
 ###### 2018-11-16
  * Fixes for GridSearch optimizer.
 
  * Include documentation with release.
 
-### ensmallen 1.10.0
+### ensmallen 1.10.0: "Corporate Catabolism"
 ###### 2018-10-20
  * Initial release.
 

README.md

@@ -1,10 +1,14 @@
-**ensmallen** is a C++ header-only library for mathematical optimization.
+<h2 align="center">
+  <a href="http://ensmallen.org/"><img src="http://ensmallen.org/img/ensmallen_text.svg" style="background-color:rgba(0,0,0,0);" height=230 alt="ensmallen: a C++ header-only library for numerical optimization"></a>
+</h2>
+
+**ensmallen** is a C++ header-only library for numerical optimization.
 
 Documentation and downloads: http://ensmallen.org
 
 ensmallen provides a simple set of abstractions for writing an objective
 function to optimize. It also provides a large set of standard and cutting-edge
-optimizers that can be used for virtually any mathematical optimization task.
+optimizers that can be used for virtually any numerical optimization task.
 These include full-batch gradient descent techniques, small-batch techniques,
 gradient-free optimizers, and constrained optimization.
 
@@ -16,14 +20,21 @@ gradient-free optimizers, and constrained optimization.
 * OpenBLAS or Intel MKL or LAPACK (see Armadillo site for details)
 
 
+### Installation
+
+ensmallen can be installed with CMake 3.3 or later.
+If CMake is not already available on your system, it can be obtained from https://cmake.org
+
+If you are using an older system such as RHEL 7 or CentOS 7,
+an updated version of CMake is also available via the EPEL repository via the `cmake3` package.
+
+
 ### License
 
-Unless stated otherwise, the source code for **ensmallen**
-is licensed under the 3-clause BSD license (the "License").
-A copy of the License is included in the "LICENSE.txt" file.
-You may also obtain a copy of the License at
-http://opensource.org/licenses/BSD-3-Clause
-
+Unless stated otherwise, the source code for **ensmallen** is licensed under the
+3-clause BSD license (the "License").  A copy of the License is included in the
+"LICENSE.txt" file.  You may also obtain a copy of the License at
+http://opensource.org/licenses/BSD-3-Clause .
 
 ### Citation
 
@@ -31,10 +42,30 @@ Please cite the following paper if you use ensmallen in your research and/or
 software. Citations are useful for the continued development and maintenance of
 the library.
 
-* S. Bhardwaj, R. Curtin, M. Edel, Y. Mentekidis, C. Sanderson.  
-  [ensmallen: a flexible C++ library for efficient function optimization](http://www.ensmallen.org/files/ensmallen_2018.pdf).  
+* S. Bhardwaj, R. Curtin, M. Edel, Y. Mentekidis, C. Sanderson.
+  [ensmallen: a flexible C++ library for efficient function optimization](http://www.ensmallen.org/files/ensmallen_2018.pdf).
   Workshop on Systems for ML and Open Source Software at NIPS 2018.
 
+```
+@article{DBLP:journals/corr/abs-1810-09361,
+  author    = {Shikhar Bhardwaj and
+               Ryan R. Curtin and
+               Marcus Edel and
+               Yannis Mentekidis and
+               Conrad Sanderson},
+  title     = {ensmallen: a flexible {C++} library for efficient function optimization},
+  journal   = {CoRR},
+  volume    = {abs/1810.09361},
+  doi       = {10.5281/zenodo.2008650},
+  year      = {2018},
+  url       = {http://arxiv.org/abs/1810.09361},
+  archivePrefix = {arXiv},
+  eprint    = {1810.09361},
+  timestamp = {Wed, 31 Oct 2018 14:24:29 +0100},
+  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1810-09361},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```
 
 ### Developers and Contributors
 
@@ -63,3 +94,5 @@ the library.
 * Conrad Sanderson
 * Dan Timson
 * N Rajiv Vaidyanathan
+* Roberto Hueso
+* Sayan Goswami

doc/callbacks.md

@@ -0,0 +1,576 @@
+Callbacks in ensmallen are methods that are called at various states during the
+optimization process, which can be used to implement and control behaviors such
+as:
+
+* Changing the learning rate.
+* Printing of the current objective.
+* Sending a message when the optimization hits a specific state such us a minimal objective.
+
+Callbacks can be passed as an argument to the `Optimize()` function:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+MomentumSGD optimizer(0.01, 32, 100000, 1e-5, true, MomentumUpdate(0.5));
+
+// Pass the built-in *PrintLoss* callback as the last argument to the
+// *Optimize()* function.
+optimizer.Optimize(f, coordinates, PrintLoss());
+```
+
+</details>
+
+Passing multiple callbacks is just the same as passing a single callback:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+MomentumSGD optimizer(0.01, 32, 100000, 1e-5, true, MomentumUpdate(0.5));
+
+// Pass the built-in *PrintLoss* and *EarlyStopAtMinLoss* callback as the last
+// argument to the *Optimize()* function.
+optimizer.Optimize(f, coordinates, PrintLoss(), EarlyStopAtMinLoss());
+```
+
+</details>
+
+It is also possible to pass a callback instantiation that allows accessing of
+internal callback parameters at a later state:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+MomentumSGD optimizer(0.01, 32, 100000, 1e-5, true, MomentumUpdate(0.5));
+
+// Create an instantiation of the built-in *StoreBestCoordinates* callback,
+// which will store the best objective and the corresponding model parameter
+// that can be accessed later.
+StoreBestCoordinates<> callback;
+
+// Pass an instantiation of the built-in *StoreBestCoordinates* callback as the
+// last argument to the *Optimize()* function.
+optimizer.Optimize(f, coordinates, callback);
+
+// Print the minimum objective that is stored inside the *StoreBestCoordinates*
+// callback that was passed to the *Optimize()* call.
+std::cout << callback.BestObjective() << std::endl;
+```
+
+</details>
+
+## Built-in Callbacks
+
+### EarlyStopAtMinLoss
+
+Stops the optimization process if the loss stops decreasing or no improvement
+has been made.
+
+#### Constructors
+
+ * `EarlyStopAtMinLoss()`
+ * `EarlyStopAtMinLoss(`_`patience`_`)`
+ * `EarlyStopAtMinLoss(`_`func`_`)`
+ * `EarlyStopAtMinLoss(`_`func`_`,`_`patience`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`patience`** | The number of epochs to wait after the minimum loss has been reached. | `10` |
+| `std::function<double(const arma::mat&)>` | **`func`** | A callback to return immediate loss evaluated by the function. | |
+
+Note that for the `func` argument above, if a
+[different matrix type](#alternate-matrix-types) is desired, instead of using
+the class `EarlyStopAtMinLoss`, the class `EarlyStopAtMinLossType<MatType>`
+should be used.
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, EarlyStopAtMinLoss());
+```
+Another example of using lambda in the constructor.
+
+```c++
+// Generate random training data and labels.
+arma::mat trainingData(5, 100, arma::fill::randu);
+arma::Row<size_t> trainingLabels =
+    arma::randi<arma::Row<size_t>>(100, arma::distr_param(0, 1));
+// Generate a validation set.
+arma::mat validationData(5, 100, arma::fill::randu);
+arma::Row<size_t> validationLabels =
+    arma::randi<arma::Row<size_t>>(100, arma::distr_param(0, 1));
+
+// Create a LogisticRegressionFunction for both the training and validation data.
+LogisticRegressionFunction lrfTrain(trainingData, trainingLabels);
+LogisticRegressionFunction lrfValidation(validationData, validationLabels);
+
+// Create a callback that will terminate when the validation loss starts to
+// increase.
+EarlyStopAtMinLoss cb(
+    [&](const arma::mat& coordinates)
+    {
+      // You could also, e.g., print the validation loss here to watch it converge.
+      return lrfValidation.Evaluate(coordinates);
+    });
+    
+arma::mat coordinates = lrfTrain.GetInitialPoint();
+SMORMS3 smorms3;
+smorms3.Optimize(lrfTrain, coordinates, cb);
+```
+
+</details>
+
+### PrintLoss
+
+Callback that prints loss to stdout or a specified output stream.
+
+#### Constructors
+
+ * `PrintLoss()`
+ * `PrintLoss(`_`output`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `std::ostream` | **`output`** | Ostream which receives output from this object. | `stdout` |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, PrintLoss());
+```
+
+</details>
+
+### ProgressBar
+
+Callback that prints a progress bar to stdout or a specified output stream.
+
+#### Constructors
+
+ * `ProgressBar()`
+ * `ProgressBar(`_`width`_`)`
+ * `ProgressBar(`_`width, output`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** | **default** |
+|----------|----------|-----------------|-------------|
+| `size_t` | **`width`** | Width of the bar. | `70` |
+| `std::ostream` | **`output`** | Ostream which receives output from this object. | `stdout` |
+
+#### Examples:
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+optimizer.Optimize(f, coordinates, ProgressBar());
+```
+
+</details>
+
+### StoreBestCoordinates
+
+Callback that stores the model parameter after every epoch if the objective
+decreased.
+
+#### Constructors
+
+ * `StoreBestCoordinates<`_`ModelMatType`_`>()`
+
+The _`ModelMatType`_ template parameter refers to the matrix type of the model
+parameter.
+
+#### Attributes
+
+The stored model parameter can be accessed via the member method
+`BestCoordinates()` and the best objective via `BestObjective()`.
+
+#### Examples:
+
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 1000, 1e-9, true);
+
+RosenbrockFunction f;
+arma::mat coordinates = f.GetInitialPoint();
+
+StoreBestCoordinates<arma::mat> cb;
+optimizer.Optimize(f, coordinates, cb);
+
+std::cout << "The optimized model found by AdaDelta has the "
+      << "parameters " << cb.BestCoordinatest();
+```
+
+</details>
+
+## Callback States
+
+Callbacks are called at several states during the optimization process:
+
+* At the beginning and end of the optimization process.
+* After any call to `Evaluate()` and `EvaluateConstraint`.
+* After any call to `Gradient()` and `GradientConstraint`.
+* At the start and end of an epoch.
+
+Each callback provides optimization relevant information that can be accessed or
+modified.
+
+### BeginOptimization
+
+Called at the beginning of the optimization process.
+
+ * `BeginOptimization(`_`optimizer, function, coordinates`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+
+### EndOptimization
+
+Called at the end of the optimization process.
+
+ * `EndOptimization(`_`optimizer, function, coordinates`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+
+### Evaluate
+
+Called after any call to `Evaluate()`.
+
+ * `Evaluate(`_`optimizer, function, coordinates, objective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `double` | **`objective`** | Objective value of the current point. |
+
+### EvaluateConstraint
+
+ Called after any call to `EvaluateConstraint()`.
+
+ * `EvaluateConstraint(`_`optimizer, function, coordinates, constraint, constraintValue`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `size_t` | **`constraint`** | The index of the constraint. |
+| `double` | **`constraintValue`** | Constraint value of the current point. |
+
+### Gradient
+
+ Called after any call to `Gradient()`.
+
+ * `Gradient(`_`optimizer, function, coordinates, gradient`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `GradType` | **`gradient`** | Matrix that holds the gradient. |
+
+### GradientConstraint
+
+ Called after any call to `GradientConstraint()`.
+
+ * `GradientConstraint(`_`optimizer, function, coordinates, constraint, gradient`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `size_t` | **`constraint`** | The index of the constraint. |
+| `GradType` | **`gradient`** | Matrix that holds the gradient. |
+
+### BeginEpoch
+
+Called at the beginning of a pass over the data. The objective may be exact or
+an estimate depending on `exactObjective` value.
+
+ * `BeginEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `size_t` | **`epoch`** | The index of the current epoch. |
+| `double` | **`objective`** | Objective value of the current point. |
+
+### EndEpoch
+
+Called at the end of a pass over the data. The objective may be exact or
+an estimate depending on `exactObjective` value.
+
+ * `EndEpoch(`_`optimizer, function, coordinates, epoch, objective`_`)`
+
+#### Attributes
+
+| **type** | **name** | **description** |
+|----------|----------|-----------------|
+| `OptimizerType` | **`optimizer`** | The optimizer used to update the function. |
+| `FunctionType` | **`function`** | The function to be optimized. |
+| `MatType` | **`coordinates`** | The current function parameter. |
+| `size_t` | **`epoch`** | The index of the current epoch. |
+| `double` | **`objective`** | Objective value of the current point. |
+
+## Custom Callbacks
+
+### Learning rate scheduling
+
+Setting the learning rate is crucially important when training because it
+controls both the speed of convergence and the ultimate performance of the
+model. One of the simplest learning rate strategies is to have a fixed learning
+rate throughout the training process. Choosing a small learning rate allows the
+optimizer to find good solutions, but this comes at the expense of limiting the
+initial speed of convergence. To overcome this tradeoff, changing the learning
+rate as more epochs have passed is commonly done in model training. The
+`Evaluate` method in combination with the ``StepSize`` method of the optimizer
+can be used to update the variables.
+
+Example code showing how to implement a custom callback to change the learning
+rate is given below.
+
+<details>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+class ExponentialDecay
+{
+  // Set up the exponential decay learning rate scheduler with the user
+  // specified decay value.
+  ExponentialDecay(const double decay) : decay(decay), learningRate(0) { }
+
+
+  // Callback function called at the start of the optimization process.
+  // In this example we will use this to save the initial learning rate.
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginOptimization(OptimizerType& /* optimizer */,
+                         FunctionType& /* function */,
+                         MatType& /* coordinates */)
+  {
+    // Save the initial learning rate.
+    learningRate = optimizer.StepSize();
+  }
+
+  // Callback function called at the end of a pass over the data. We are only
+  // interested in the current epoch and the optimizer, we ignore the rest.
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& optimizer,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t epoch,
+                const double objective)
+  {
+    // Update the learning rate.
+    optimizer.StepSize() = learningRate * (1.0 - std::pow(decay,
+        (double) epoch));
+  }
+
+  double learningRate;
+};
+
+int main()
+{
+  // First, generate some random data, with 10000 points and 10 dimensions.
+  // This data has no pattern and as such will make a model that's not very
+  // useful---but the purpose here is just demonstration. :)
+  //
+  // For a more "real world" situation, load a dataset from file using X.load()
+  // and y.load() (but make sure the matrix is column-major, so that each
+  // observation/data point corresponds to a *column*, *not* a row.
+  arma::mat data(10, 10000, arma::fill::randn);
+  arma::rowvec responses(10000, arma::fill::randn);
+
+  // Create a starting point for our optimization randomly.  The model has 10
+  // parameters, so the shape is 10x1.
+  arma::mat startingPoint(10, 1, arma::fill::randn);
+
+  // Construct the objective function.
+  LinearRegressionFunction lrf(data, responses);
+  arma::mat lrfParams(startingPoint);
+
+  // Create the StandardSGD optimizer with specified parameters.
+  // The ens::StandardSGD type can be replaced with any ensmallen optimizer
+  //that can handle differentiable functions.
+  StandardSGD optimizer(0.001, 1, 0, 1e-15, true);
+
+  // Use the StandardSGD optimizer with specified parameters to minimize the
+  // LinearRegressionFunction and pass the *exponential decay*
+  // callback function from above.
+  optimizer.Optimize(lrf, lrfParams, ExponentialDecay(0.01));
+
+  // Print the trained model parameter.
+  std::cout << lrfParams.t();
+}
+```
+
+</details>
+
+### Early stopping at minimum loss
+
+Early stopping is a technique for controlling overfitting in machine learning
+models, especially neural networks, by stopping the optimization process before
+the model has trained for the maximum number of iterations.
+
+Example code showing how to implement a custom callback to stop the optimization
+when the minimum of loss has been reached is given below.
+
+<details>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+#include <ensmallen.hpp>
+
+// This class implements early stopping at minimum loss callback function to
+// terminate the optimization process early if the loss stops decreasing.
+class EarlyStop
+{
+ public:
+  // Set up the early stop at min loss class, which keeps track of the minimum
+  // loss.
+  EarlyStop() : bestObjective(std::numeric_limits<double>::max()) { }
+
+  // Callback function called at the end of a pass over the data, which provides
+  // the current objective. We are only interested in the objective and ignore
+  // the rest.
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double objective)
+  {
+    // Check if the given objective is lower as the previous objective.
+    if (objective < bestObjective)
+    {
+      // Update the local objective.
+      bestObjective = objective;
+    }
+    else
+    {
+      // Stop the optimization process.
+      return true;
+    }
+
+    // Do not stop the optimization process.
+    return false;
+  }
+
+  // Locally-stored best objective.
+  double bestObjective;
+};
+
+int main()
+{
+  // First, generate some random data, with 10000 points and 10 dimensions.
+  // This data has no pattern and as such will make a model that's not very
+  // useful---but the purpose here is just demonstration. :)
+  //
+  // For a more "real world" situation, load a dataset from file using X.load()
+  // and y.load() (but make sure the matrix is column-major, so that each
+  // observation/data point corresponds to a *column*, *not* a row.
+  arma::mat data(10, 10000, arma::fill::randn);
+  arma::rowvec responses(10000, arma::fill::randn);
+
+  // Create a starting point for our optimization randomly.  The model has 10
+  // parameters, so the shape is 10x1.
+  arma::mat startingPoint(10, 1, arma::fill::randn);
+
+  // Construct the objective function.
+  LinearRegressionFunction lrf(data, responses);
+  arma::mat lrfParams(startingPoint);
+
+  // Create the L_BFGS optimizer with default parameters.
+  // The ens::L_BFGS type can be replaced with any ensmallen optimizer that can
+  // handle differentiable functions.
+  ens::L_BFGS lbfgs;
+
+  // Use the L_BFGS optimizer with default parameters to minimize the
+  // LinearRegressionFunction and pass the *early stopping at minimum loss*
+  // callback function from above.
+  lbfgs.Optimize(lrf, lrfParams, EarlyStop());
+
+  // Print the trained model parameter.
+  std::cout << lrfParams.t();
+}
+```
+
+</details>
+
+Note that we have simply passed an instantiation of `EarlyStop` the
+rest is handled inside the optimizer.
+
+ensmallen provides a more complete and general implementation of a
+[early stopping](#EarlyStopAtMinLoss) at minimum loss callback function.

doc/function_types.md

@@ -4,6 +4,10 @@ The least restrictive type of function that can be implemented in ensmallen is
 a function for which only the objective can be evaluated.  For this, a class
 with the following API must be implemented:
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 class ArbitraryFunctionType
 {
@@ -13,6 +17,8 @@ class ArbitraryFunctionType
 };
 ```
 
+</details>
+
 For this type of function, we assume that the gradient `f'(x)` is not
 computable.  If it is, see [differentiable functions](#differentiable-functions).
 
@@ -23,6 +29,9 @@ The following optimizers can be used to optimize an arbitrary function:
 
  - [Simulated Annealing](#simulated-annealing-sa)
  - [CNE](#cne)
+ - [DE](#de)
+ - [PSO](#pso)
+ - [SPSA](#simultaneous-perturbation-stochastic-approximation-spsa)
 
 Each of these optimizers has an `Optimize()` function that is called as
 `Optimize(f, x)` where `f` is the function to be optimized (which implements
@@ -30,11 +39,15 @@ Each of these optimizers has an `Optimize()` function that is called as
 `Optimize()` is called, `x` will hold the final result of the optimization
 (that is, the best `x` found that minimizes `f(x)`).
 
-#### Example: Linear Regression
+#### Example: squared function optimization
 
 An example program that implements the objective function f(x) = 2 |x|^2 is
 shown below, using the simulated annealing optimizer.
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 #include <ensmallen.hpp>
 
@@ -50,7 +63,7 @@ class SquaredFunction
 
 int main()
 {
-  // The minimum is at x = [0 0 0].  Our initial point is chosen to be 
+  // The minimum is at x = [0 0 0].  Our initial point is chosen to be
   // [1.0, -1.0, 1.0].
   arma::mat x("1.0 -1.0 1.0");
 
@@ -66,6 +79,8 @@ int main()
 }
 ```
 
+</details>
+
 ## Differentiable functions
 
 Probably the most common type of function that can be optimized with ensmallen
@@ -73,6 +88,10 @@ is a differentiable function, where both f(x) and f'(x) can be calculated.  To
 optimize a differentiable function with ensmallen, a class must be implemented
 that follows the API below:
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 class DifferentiableFunctionType
 {
@@ -97,6 +116,8 @@ class DifferentiableFunctionType
 };
 ```
 
+</details>
+
 Note that you may implement *either* `Evaluate()` and `Gradient()` *or*
 `EvaluateWithGradient()`, but it is not mandatory to implement both.  (Of
 course, supplying both is okay too.)  It often results in faster code when
@@ -134,6 +155,10 @@ In the example program, we optimize this objective function and compare the
 runtime of an implementation that uses `Evaluate()` and `Gradient()`, and the
 runtime of an implementation that uses `EvaluateWithGradient()`.
 
+<details>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 #include <ensmallen.hpp>
 
@@ -144,7 +169,7 @@ class LinearRegressionFunction
  public:
   // Construct the object with the given data matrix and responses.
   LinearRegressionFunction(const arma::mat& dataIn,
-                           const arma::rowvec& responsesIn) : 
+                           const arma::rowvec& responsesIn) :
       data(dataIn), responses(responsesIn) { }
 
   // Return the objective function for model parameters x.
@@ -171,13 +196,13 @@ class LinearRegressionEWGFunction
 {
  public:
   // Construct the object with the given data matrix and responses.
-  LinearRegressionEWGFunction(const arma::mat& dataIn, 
-                              const arma::rowvec& responsesIn) : 
+  LinearRegressionEWGFunction(const arma::mat& dataIn,
+                              const arma::rowvec& responsesIn) :
       data(dataIn), responses(responsesIn) { }
 
   // Simultaneously compute both the objective function and gradient for model
   // parameters x.  Note that this is faster than implementing Evaluate() and
-  // Gradient() individually because it caches the computation of 
+  // Gradient() individually because it caches the computation of
   // (responses - x.t() * data)!
   double EvaluateWithGradient(const arma::mat& x, arma::mat& g)
   {
@@ -246,6 +271,8 @@ int main()
 }
 ```
 
+</details>
+
 ### Partially differentiable functions
 
 Some differentiable functions have the additional property that the gradient
@@ -259,6 +286,10 @@ useful for coordinate descent type algorithms.
 To use ensmallen optimizers to minimize these types of functions, only two
 functions needs to be added to the differentiable function type:
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 // Compute the partial gradient f'_j(x) with respect to data coordinate j and
 // store it in the sparse matrix g.
@@ -268,6 +299,8 @@ void Gradient(const arma::mat& x, const size_t j, arma::sp_mat& g);
 size_t NumFeatures();
 ```
 
+</details>
+
 **Note**: many partially differentiable function optimizers do not require a
 regular implementation of the `Gradient()`, so that function may be omitted.
 
@@ -286,13 +319,17 @@ f(x) = f_0(x) + f_1(x) + ... + f_N(x).
 ```
 
 In this function type, we assume the gradient `f'(x)` is not computable.  If it
-is, see [#Differentiable-separable-functions].
+is, see [differentiable separable functions](#differentiable-separable-functions).
 
 For machine learning tasks, the objective function may be, e.g., the sum of a
 function taken across many data points.  Implementing an arbitrary separable
 function type in ensmallen is similar to implementing an arbitrary objective
 function, but with a few extra utility methods:
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 class ArbitrarySeparableFunctionType
 {
@@ -314,6 +351,8 @@ class ArbitrarySeparableFunctionType
 };
 ```
 
+</details>
+
 Each of the implemented methods is allowed to have additional cv-modifiers
 (`static`, `const`, etc.).
 
@@ -333,7 +372,7 @@ intensive operation for your objective function, it may be best to avoid using
 a non-separable arbitrary function optimizer.
 
 **Note**: if possible, it's often better to try and use a gradient-based
-approach.  See [#Differentiable-separable-functions]
+approach.  See [differentiable separable functions](#differentiable-separable-functions)
 for separable f(x) where the gradient f'(x) can be computed.
 
 The example program below demonstrates the implementation and use of an
@@ -347,6 +386,10 @@ $$ f_i(x) = (\operatorname{responses}(i) - x' * \operatorname{data}(i))^2 $$
 where $\operatorname{data}(i)$ represents the data point indexed by $i$ and
 $\operatorname{responses}(i)$ represents the observed response indexed by $i$.
 
+<details>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 #include <ensmallen.hpp>
 
@@ -423,6 +466,8 @@ int main()
 }
 ```
 
+</details>
+
 ## Differentiable separable functions
 
 Likely the most important type of function to be optimized in machine learning
@@ -445,6 +490,10 @@ function taken across many data points.  Implementing a differentiable
 separable function type in ensmallen is similar to implementing an ordinary
 differentiable function, but with a few extra utility methods:
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 class ArbitrarySeparableFunctionType
 {
@@ -481,7 +530,7 @@ class ArbitrarySeparableFunctionType
   //
   // Given parameters x and a matrix g, return the sum of the individual
   // functions f_i(x) + ... + f_{i + batchSize - 1}(x), and store the sum of
-  // the gradient of individual functions f'_i(x) + ... + 
+  // the gradient of individual functions f'_i(x) + ... +
   // f'_{i + batchSize - 1}(x) into the provided matrix g.  g should have the
   // same size (rows, columns) as x.  i will always be greater than 0, and i +
   // batchSize will be less than or equal to the value of NumFunctions().
@@ -492,6 +541,8 @@ class ArbitrarySeparableFunctionType
 };
 ```
 
+</details>
+
 Note that you may implement *either* `Evaluate()` and `Gradient()` *or*
 `EvaluateWithGradient()`, but it is not mandatory to implement both.  (Of
 course, supplying both is okay too.)  It often results in faster code when
@@ -501,29 +552,38 @@ and f'(x) compute some of the same intermediate quantities.
 Each of the implemented methods is allowed to have additional cv-modifiers
 (`static`, `const`, etc.).
 
-The following optimizers can be used with differentiable functions:
+The following optimizers can be used with differentiable separable functions:
 
+ - [AdaBound](#adabound)
  - [AdaDelta](#adadelta)
  - [AdaGrad](#adagrad)
  - [Adam](#adam)
  - [AdaMax](#adamax)
+ - [AMSBound](#amsbound)
  - [AMSGrad](#amsgrad)
  - [Big Batch SGD](#big-batch-sgd)
+ - [Eve](#eve)
+ - [FTML](#ftml-follow-the-moving-leader)
  - [IQN](#iqn)
  - [Katyusha](#katyusha)
+ - [Lookahead](#lookahead)
  - [Momentum SGD](#momentum-sgd)
  - [Nadam](#nadam)
  - [NadaMax](#nadamax)
  - [NesterovMomentumSGD](#nesterov-momentum-sgd)
  - [OptimisticAdam](#optimisticadam)
+ - [QHAdam](#qhadam)
+ - [QHSGD](#qhsgd)
  - [RMSProp](#rmsprop)
  - [SARAH/SARAH+](#stochastic-recursive-gradient-algorithm-sarahsarah)
  - [SGD](#standard-sgd)
  - [Stochastic Gradient Descent with Restarts (SGDR)](#stochastic-gradient-descent-with-restarts-sgdr)
  - [Snapshot SGDR](#snapshot-stochastic-gradient-descent-with-restarts)
  - [SMORMS3](#smorms3)
- - [SVRG](#standard-stochastic-variance-reduced-gradient-svrg)
  - [SPALeRA](#spalera-stochastic-gradient-descent-spalerasgd)
+ - [SWATS](#swats)
+ - [SVRG](#standard-stochastic-variance-reduced-gradient-svrg)
+ - [WNGrad](#wngrad)
 
 The example program below demonstrates the implementation and use of an
 arbitrary separable function.  The function used is the linear regression
@@ -540,6 +600,10 @@ represents the observed response indexed by `i`.  This example implementation
 only implements `EvaluateWithGradient()` in order to avoid redundant
 calculations.
 
+<details>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 #include <ensmallen.hpp>
 
@@ -623,12 +687,18 @@ int main()
 }
 ```
 
+</details>
+
 ### Sparse differentiable separable functions
 
 Some differentiable separable functions have the additional property that
 the gradient `f'_i(x)` is sparse.  When this is true, one additional method can
 be implemented as part of the class to be optimized:
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 // Add this definition to use sparse differentiable separable function
 // optimizers.  Given x, store the sum of the sparse gradient f'_i(x) + ... +
@@ -639,9 +709,15 @@ void Gradient(const arma::mat& x,
               const size_t batchSize);
 ```
 
+</details>
+
 It's also possible to instead use templates to provide only one `Gradient()`
 function for both sparse and non-sparse optimizers:
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 // This provides Gradient() for both sparse and non-sparse optimizers.
 template<typename GradType>
@@ -651,6 +727,8 @@ void Gradient(const arma::mat& x,
               const size_t batchSize);
 ```
 
+</details>
+
 If either of these methods are available, then any ensmallen optimizer that
 optimizes sparse separable differentiable functions may be used.  This
 includes:
@@ -668,6 +746,10 @@ an `ArbitraryFunctionType`---but for any categorical dimension `x_i` in `x`, the
 value will be in the range [0, c_i - 1] where `c_i` is the number of categories
 in dimension `x_i`.
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 class CategoricalFunction
 {
@@ -677,6 +759,8 @@ class CategoricalFunction
 };
 ```
 
+</details>
+
 However, when an optimizer's Optimize() method is called, two additional
 parameters must be specified, in addition to the function to optimize and the
 matrix holding the parameters:
@@ -696,6 +780,10 @@ The following optimizers can be used in this way to optimize a categorical funct
 
 An example program showing usage of categorical optimization is shown below.
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 #include <ensmallen.hpp>
 
@@ -751,27 +839,82 @@ int main()
 }
 ```
 
+</details>
+
+
+## Multi-objective functions
+
+A multi-objective optimizer does not return just one set of coordinates at the
+minimum of all objective functions, but instead finds a *front* or *frontier* of
+possible coordinates that are Pareto-optimal (that is, no individual objective
+function's value can be reduced without increasing at least one other
+objective function).
+
+In order to optimize a multi-objective function with ensmallen, a `std::tuple<>`
+containing multiple `ArbitraryFunctionType`s ([see here](#arbitrary-functions))
+should be passed to a multi-objective optimizer's `Optimize()` function.
+
+An example below simultaneously optimizes the generalized Rosenbrock function
+in 6 dimensions and the Wood function using [NSGA2](#nsga2).
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+GeneralizedRosenbrockFunction rf(6);
+WoodFunction wf;
+std::tuple<GeneralizedRosenbrockFunction, WoodFunction> objectives(rf, wf);
+
+// Create an initial point (a random point in 6 dimensions).
+arma::mat coordinates(6, 1, arma::fill::randu);
+
+// `coordinates` will be set to the coordinates on the best front that minimize the
+// sum of objective functions, and `bestFrontSum` will be the sum of all objectives
+// at that coordinate set.
+NSGA2 nsga;
+double bestFrontSum = nsga.Optimize(objectives, coordinates);
+
+// Set `bestFront` to contain all of the coordinates on the best front.
+std::vector<arma::mat> bestFront = optimizer.Front();
+}
+```
+
+</details>
+
+*Note*: all multi-objective function optimizers have both the function `Optimize()` to find the
+best front, and also the function `Front()` to return all sets of coordinates that are on the
+front.
+
+The following optimizers can be used with multi-objective functions:
+- [NSGA2](#nsga2)
+
 ## Constrained functions
 
 A constrained function is an objective function `f(x)` that is also subject to
 some constraints on `x`.  (For instance, perhaps a constraint could be that `x`
 is a positive semidefinite matrix.)  ensmallen is able to handle differentiable
 objective functions of this type---so, `f'(x)` must also be computable.  Given
-some set of constraints c_0(x), ... c_M(x), we can re-express our constrained
+some set of constraints `c_0(x)`, ..., `c_M(x)`, we can re-express our constrained
 objective function as
 
 ```
 f_C(x) = f(x) + c_0(x) + ... + c_M(x)
 ```
 
-where the constraint `c_i(x)` is `DBL_MAX` if it is not satisfied, and
-otherwise takes some real value.  For a "hard constraint", we can simply take
-`c_i(x) = 0` when it is satisfied.  But allowing `c_i(x)` to return anything
-allows us to handle "soft" constraints also.
+where the (soft) constraint `c_i(x)` is a positive value if it is not satisfied, and
+`0` if it is satisfied.  The soft constraint `c_i(x)` should take some value
+representing how far from a feasible solution `x` is.  It should be
+differentiable, since ensmallen's constrained optimizers will use the gradient
+of the constraint to find a feasible solution.
 
 In order to optimize a constrained function with ensmallen, a class
 implementing the API below is required.
 
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 class ConstrainedFunctionType
 {
@@ -786,21 +929,21 @@ class ConstrainedFunctionType
   size_t NumConstraints();
 
   // Evaluate constraint i at the parameters x.  If the constraint is
-  // unsatisfied, DBL_MAX should be returned.  If the constraint is satisfied,
-  // any real value can be returned.  The optimizer will add this value to its
-  // overall objective that it is trying to minimize.  (So, a hard constraint
-  // can just return 0 if it's satisfied.)
+  // unsatisfied, a value greater than 0 should be returned.  If the constraint
+  // is satisfied, 0 should be returned.  The optimizer will add this value to
+  // its overall objective that it is trying to minimize.
   double EvaluateConstraint(const size_t i, const arma::mat& x);
 
   // Evaluate the gradient of constraint i at the parameters x, storing the
-  // result in the given matrix g.  If this is a hard constraint you can set
-  // the gradient to 0.  If the constraint is not satisfied, it could be
-  // helpful to set the gradient in such a way that the gradient points in the
+  // result in the given matrix g.  If the constraint is not satisfied, the
+  // gradient should be set in such a way that the gradient points in the
   // direction where the constraint would be satisfied.
   void GradientConstraint(const size_t i, const arma::mat& x, arma::mat& g);
 };
 ```
 
+</details>
+
 A constrained function can be optimized with the following optimizers:
 
  - [Augmented Lagrangian](#augmented-lagrangian)
@@ -843,10 +986,14 @@ solver.  The list of SDP solvers is below:
 
 Example code showing how to solve an SDP is given below.
 
+<details>
+<summary>Click to collapse/expand example code.
+</summary>
+
 ```c++
 int main()
 {
-  // We will build a toy semidefinite program and then use the PrimalDualSolver to find a solution 
+  // We will build a toy semidefinite program and then use the PrimalDualSolver to find a solution
 
   // The semi-definite constraint looks like:
   //
@@ -936,13 +1083,111 @@ int main()
   // use the PrimalDualSolver to solve it.
   // ens::PrimalDualSolver could be replaced with ens::LRSDP or other ensmallen
   // SDP solvers.
-  PrimalDualSolver<SDP<arma::sp_mat>> solver(sdp);
+  PrimalDualSolver solver;
   arma::mat X, Z;
   arma::vec ysparse, ydense;
   // ysparse, ydense, and Z hold the primal and dual variables found during the
   // optimization.
-  const double obj = solver.Optimize(X, ysparse, ydense, Z);
+  const double obj = solver.Optimize(sdp, X, ysparse, ydense, Z);
 
   std::cout << "SDP optimized with objective " << obj << "." << std::endl;
 }
 ```
+
+</details>
+
+## Alternate matrix types
+
+All of the examples above (and throughout the rest of the documentation)
+generally assume that the matrix being optimized has type `arma::mat`.  But
+ensmallen's optimizers are capable of optimizing more types than just dense
+Armadillo matrices.  In fact, the full signature of each optimizer's
+`Optimize()` method is this:
+
+```
+template<typename FunctionType, typename MatType>
+typename MatType::elem_type Optimize(FunctionType& function,
+                                     MatType& coordinates);
+```
+
+The return type, `typename MatType::elem_type`, is just the numeric type held by
+the given matrix type.  So, for `arma::mat`, the return type is just `double`.
+In addition, optimizers for differentiable functions have a third template
+parameter, `GradType`, which specifies the type of the gradient.  `GradType` can
+be manually specified in the situation where, e.g., a sparse gradient is
+desired.
+
+It is easy to write a function to optimize, e.g., an `arma::fmat`.  Here is an
+example, adapted from the `SquaredFunction` example from the
+[arbitrary function documentation](#example__squared_function_optimization).
+
+<details open>
+<summary>Click to collapse/expand example code.
+</summary>
+
+```c++
+#include <ensmallen.hpp>
+
+class SquaredFunction
+{
+ public:
+  // This returns f(x) = 2 |x|^2.
+  float Evaluate(const arma::fmat& x)
+  {
+    return 2 * std::pow(arma::norm(x), 2.0);
+  }
+
+  void Gradient(const arma::fmat& x, arma::fmat& gradient)
+  {
+    gradient = 4 * x;
+  }
+};
+
+int main()
+{
+  // The minimum is at x = [0 0 0].  Our initial point is chosen to be
+  // [1.0, -1.0, 1.0].
+  arma::fmat x("1.0 -1.0 1.0");
+
+  // Create simulated annealing optimizer with default options.
+  // The ens::SA<> type can be replaced with any suitable ensmallen optimizer
+  // that is able to handle arbitrary functions.
+  ens::L_BFGS optimizer;
+  SquaredFunction f; // Create function to be optimized.
+  optimizer.Optimize(f, x); // The optimizer will infer arma::fmat!
+
+  std::cout << "Minimum of squared function found with simulated annealing is "
+      << x;
+}
+```
+
+</details>
+
+Note that we have simply changed the `SquaredFunction` to accept `arma::fmat`
+instead of `arma::mat` as parameters to `Evaluate()`, and the return type has
+accordingly been changed to `float` from `double`.  It would even be possible to
+optimize functions with sparse coordinates by having `Evaluate()` take a sparse
+matrix (i.e. `arma::sp_mat`).
+
+If it were desired to represent the gradient as a sparse type, the `Gradient()`
+function would need to be modified to take a sparse matrix (i.e. `arma::sp_mat`
+or similar), and then you could call `optimizer.Optimize<SquaredFunction,
+arma::mat, arma::sp_mat>(f, x);` to perform the optimization while using sparse
+matrix types to represent the gradient.  Using sparse `MatType` or `GradType`
+should *only* be done when it is known that the objective matrix and/or
+gradients will be sparse; otherwise the code may run very slow!
+
+ensmallen will automatically infer `MatType` from the call to `Optimize()`, and
+check that the given `FunctionType` has all of the necessary functions for the
+given `MatType`, throwing a `static_assert` error if not.  If you would like to
+disable these checks, define the macro `ENS_DISABLE_TYPE_CHECKS` before
+including ensmallen:
+
+```
+#define ENS_DISABLE_TYPE_CHECKS
+#include <ensmallen.hpp>
+```
+
+This can be useful for situations where you know that the checks should be
+ignored.  However, be aware that the code may fail to compile and give more
+confusing and difficult error messages!

include/ensmallen.hpp

@@ -29,24 +29,28 @@
   #error "please enable C++11/C++14 mode in your compiler"
 #endif
 
-#if ((ARMA_VERSION_MAJOR < 6) || ((ARMA_VERSION_MAJOR == 6) && (ARMA_VERSION_MINOR < 500)))
-  #error "need Armadillo version 6.500 or later"
+#if ((ARMA_VERSION_MAJOR < 8) || ((ARMA_VERSION_MAJOR == 8) && (ARMA_VERSION_MINOR < 400)))
+  #error "need Armadillo version 8.400 or later"
 #endif
 
-#include <cmath>
-#include <cstdlib>
-#include <cstdio>
-#include <cstring>
 #include <cctype>
-#include <climits>
 #include <cfloat>
+#include <climits>
+#include <cmath>
 #include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <map>
+#include <set>
+#include <limits>
+#include <sstream>
 #include <stdexcept>
+#include <string>
 #include <tuple>
 #include <utility>
-#include <iostream>
-#include <string>
-#include <sstream>
+#include <vector>
 
 // On Visual Studio, disable C4519 (default arguments for function templates)
 // since it's by default an error, which doesn't even make any sense because
@@ -59,15 +63,29 @@
 #include "ensmallen_bits/ens_version.hpp"
 #include "ensmallen_bits/log.hpp" // TODO: should move to another place
 
+#include "ensmallen_bits/utility/any.hpp"
+#include "ensmallen_bits/utility/arma_traits.hpp"
+
+// Callbacks.
+#include "ensmallen_bits/callbacks/callbacks.hpp"
+#include "ensmallen_bits/callbacks/early_stop_at_min_loss.hpp"
+#include "ensmallen_bits/callbacks/print_loss.hpp"
+#include "ensmallen_bits/callbacks/progress_bar.hpp"
+#include "ensmallen_bits/callbacks/store_best_coordinates.hpp"
+#include "ensmallen_bits/callbacks/timer_stop.hpp"
+
 #include "ensmallen_bits/problems/problems.hpp" // TODO: should move to another place
 
+#include "ensmallen_bits/ada_bound/ada_bound.hpp"
 #include "ensmallen_bits/ada_delta/ada_delta.hpp"
 #include "ensmallen_bits/ada_grad/ada_grad.hpp"
 #include "ensmallen_bits/adam/adam.hpp"
+#include "ensmallen_bits/qhadam/qhadam.hpp"
 #include "ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp"
 #include "ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp"
 #include "ensmallen_bits/cmaes/cmaes.hpp"
 #include "ensmallen_bits/cne/cne.hpp"
+#include "ensmallen_bits/de/de.hpp"
 #include "ensmallen_bits/eve/eve.hpp"
 #include "ensmallen_bits/ftml/ftml.hpp"
 
@@ -79,8 +97,11 @@
 #include "ensmallen_bits/iqn/iqn.hpp"
 #include "ensmallen_bits/katyusha/katyusha.hpp"
 #include "ensmallen_bits/lbfgs/lbfgs.hpp"
+#include "ensmallen_bits/lookahead/lookahead.hpp"
+#include "ensmallen_bits/nsga2/nsga2.hpp"
 #include "ensmallen_bits/padam/padam.hpp"
 #include "ensmallen_bits/parallel_sgd/parallel_sgd.hpp"
+#include "ensmallen_bits/pso/pso.hpp"
 #include "ensmallen_bits/rmsprop/rmsprop.hpp"
 
 #include "ensmallen_bits/sa/sa.hpp"

include/ensmallen_bits/ada_bound/ada_bound.hpp

@@ -0,0 +1,209 @@
+/**
+ * @file ada_bound.hpp
+ * @author Marcus Edel
+ *
+ * Class wrapper for the AdaBound and AMSBound update Policy.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BOUND_HPP
+#define ENSMALLEN_ADA_BOUND_HPP
+
+#include <ensmallen_bits/sgd/sgd.hpp>
+#include "ada_bound_update.hpp"
+#include "ams_bound_update.hpp"
+
+namespace ens {
+
+/**
+ * AdaBound and AMSBound, employ dynamic bounds on learning rates to achieve a
+ * gradual and smooth transition from adaptive methods to SGD.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{Luo2019AdaBound,
+ *   author    = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
+ *   title     = {Adaptive Gradient Methods with Dynamic Bound of Learning
+ *                Rate},
+ *   booktitle = {Proceedings of the 7th International Conference on Learning
+ *                Representations},
+ *   month     = {May},
+ *   year      = {2019},
+ *   address   = {New Orleans, Louisiana}
+ * }
+ * @endcode
+ *
+ * AdaBound and AMSBound can optimize differentiable separable functions.
+ * For more details, see the documentation on function types included with this
+ * distribution or on the ensmallen website.
+ */
+template<typename UpdatePolicyType = AdaBoundUpdate,
+         typename DecayPolicyType = NoDecay>
+class AdaBoundType
+{
+ public:
+  /**
+   * Construct the AdaBoundType optimizer with the given function and
+   * parameters. AdaBoundType is sensitive to its parameters and hence a good
+   * hyper paramater selection is necessary as its default may not fit every
+   * case.
+   *
+   * The maximum number of iterations refers to the maximum number of
+   * points that are processed (i.e., one iteration equals one point; one
+   * iteration does not equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param finalLr The final (SGD) learning rate.
+   * @param gamma The convergence speed of the bound functions.
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+   *     estimates.
+   * @param epsilon Value used to initialise the mean squared gradient
+   *     parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *     limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *     function is visited in linear order.
+   * @param resetPolicy If true, parameters are reset before every Optimize
+   *     call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
+   */
+  AdaBoundType(const double stepSize = 0.001,
+               const size_t batchSize = 32,
+               const double finalLr = 0.1,
+               const double gamma = 1e-3,
+               const double beta1 = 0.9,
+               const double beta2 = 0.999,
+               const double epsilon = 1e-8,
+               const size_t maxIterations = 100000,
+               const double tolerance = 1e-5,
+               const bool shuffle = true,
+               const bool resetPolicy = true,
+               const bool exactObjective = false);
+
+  /**
+   * Optimize the given function using AdaBoundType. The given starting point
+   * will be modified to store the finishing point of the algorithm, and the
+   * final objective value is returned. The DecomposableFunctionType is checked
+   * for API consistency at compile time.
+   *
+   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(DecomposableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
+  {
+    return optimizer.template Optimize<DecomposableFunctionType, MatType,
+        GradType, CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename DecomposableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(DecomposableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<DecomposableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Get the final (SGD) learning rate.
+  double FinalLr() const { return optimizer.UpdatePolicy().FinalLr(); }
+  //! Modify the final (SGD) learning rate.
+  double& FinalLr() { return optimizer.UpdatePolicy().FinalLr(); }
+
+  //! Get the convergence speed of the bound functions.
+  double Gamma() const { return optimizer.UpdatePolicy().Gamma(); }
+  //! Modify the convergence speed of the bound functions.
+  double& Gamma() { return optimizer.UpdatePolicy().Gamma(); }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
+
+  //! Get the value used to initialise the mean squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the mean squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
+  //! Get whether or not the update policy parameters are reset before
+  //! Optimize call.
+  bool ResetPolicy() const { return optimizer.ResetPolicy(); }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return optimizer.ResetPolicy(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with QHAdam policy.
+  SGD<UpdatePolicyType, DecayPolicyType> optimizer;
+};
+
+using AdaBound = AdaBoundType<AdaBoundUpdate>;
+using AMSBound = AdaBoundType<AMSBoundUpdate>;
+
+} // namespace ens
+
+// Include implementation.
+#include "ada_bound_impl.hpp"
+
+#endif

include/ensmallen_bits/ada_bound/ada_bound_impl.hpp

@@ -0,0 +1,47 @@
+/**
+ * @file qhadam_impl.hpp
+ * @author Niteya Shah
+ *
+ * Implementation of QHAdam class wrapper.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BOUND_IMPL_HPP
+#define ENSMALLEN_ADA_BOUND_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "ada_bound.hpp"
+
+namespace ens {
+
+template<typename UpdatePolicyType, typename DecayPolicyType>
+AdaBoundType<UpdatePolicyType, DecayPolicyType>::AdaBoundType(
+    const double stepSize,
+    const size_t batchSize,
+    const double finalLr,
+    const double gamma,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle,
+    const bool resetPolicy,
+    const bool exactObjective) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              UpdatePolicyType(finalLr, gamma, epsilon, beta1, beta2),
+              NoDecay(),
+              resetPolicy,
+              exactObjective)
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+ #endif

include/ensmallen_bits/ada_bound/ada_bound_update.hpp

@@ -0,0 +1,207 @@
+/**
+ * @file ada_bound_update.hpp
+ * @author Marcus Edel
+ *
+ * Implments the AdaBound Optimizer. AdaBound is a variant of Adam which
+ * employs dynamic bounds on learning rates.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_ADA_BOUND_UPDATE_HPP
+#define ENSMALLEN_ADA_BOUND_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * AdaBound employs dynamic bounds on learning rates to achieve a gradual and
+ * smooth transition from adaptive methods to SGD.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{Luo2019AdaBound,
+ *   author    = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
+ *   title     = {Adaptive Gradient Methods with Dynamic Bound of Learning
+ *                Rate},
+ *   booktitle = {Proceedings of the 7th International Conference on Learning
+ *                Representations},
+ *   month     = {May},
+ *   year      = {2019},
+ *   address   = {New Orleans, Louisiana}
+ * }
+ * @endcode
+ */
+class AdaBoundUpdate
+{
+ public:
+  /**
+   * Construct the AdaBound update policy with the given parameters.
+   *
+   * @param finalLr The final (SGD) learning rate.
+   * @param gamma The convergence speed of the bound functions.
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AdaBoundUpdate(const double finalLr = 0.1,
+                 const double gamma = 1e-3,
+                 const double epsilon = 1e-8,
+                 const double beta1 = 0.9,
+                 const double beta2 = 0.999) :
+    finalLr(finalLr),
+    gamma(gamma),
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the final (SGD) learning rate.
+  double FinalLr() const { return finalLr; }
+  //! Modify the final (SGD) learning rate.
+  double& FinalLr() { return finalLr; }
+
+  //! Get the convergence speed of the bound functions.
+  double Gamma() const { return finalLr; }
+  //! Modify the convergence speed of the bound functions.
+  double& Gamma() { return finalLr; }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdaBoundUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaBoundUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent), first(true), initialStepSize(0)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AdaBound.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Convenience typedefs.
+      typedef typename MatType::elem_type ElemType;
+
+      // Save the initial step size.
+      if (first)
+      {
+        first = false;
+        initialStepSize = stepSize;
+      }
+
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // Decay the first and second moment running average coefficient.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
+      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
+          parent.iteration + 1));
+      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
+          parent.iteration));
+
+       // Applies bounds on actual learning rate.
+      iterate -= arma::clamp((stepSize *
+          std::sqrt(biasCorrection2) / biasCorrection1) / (arma::sqrt(v) +
+          parent.epsilon), lower, upper) % m;
+    }
+
+   private:
+    // Instantiated parent object.
+    AdaBoundUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // Whether this is the first call of the Update method.
+    bool first;
+
+    // The initial (Adam) learning rate.
+    double initialStepSize;
+  };
+
+ private:
+  // The final (SGD) learning rate.
+  double finalLr;
+
+  // The convergence speed of the bound functions.
+  double gamma;
+
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The number of iterations.
+  size_t iteration;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ada_bound/ams_bound_update.hpp

@@ -0,0 +1,214 @@
+/**
+ * @file ams_bound_update.hpp
+ * @author Marcus Edel
+ *
+ * Implments the AMSBound Optimizer. AMSBound is a variant of Adam which
+ * employs dynamic bounds on learning rates.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_AMS_BOUND_UPDATE_HPP
+#define ENSMALLEN_AMS_BOUND_UPDATE_HPP
+
+namespace ens {
+
+/**
+ * AMSBound employs dynamic bounds on learning rates to achieve a gradual and
+ * smooth transition from adaptive methods to SGD.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @inproceedings{Luo2019AdaBound,
+ *   author    = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
+ *   title     = {Adaptive Gradient Methods with Dynamic Bound of Learning
+ *                Rate},
+ *   booktitle = {Proceedings of the 7th International Conference on Learning
+ *                Representations},
+ *   month     = {May},
+ *   year      = {2019},
+ *   address   = {New Orleans, Louisiana}
+ * }
+ * @endcode
+ */
+class AMSBoundUpdate
+{
+ public:
+  /**
+   * Construct the AMSBound update policy with the given parameters.
+   *
+   * @param finalLr The final (SGD) learning rate.
+   * @param gamma The convergence speed of the bound functions.
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AMSBoundUpdate(const double finalLr = 0.1,
+                 const double gamma = 1e-3,
+                 const double epsilon = 1e-8,
+                 const double beta1 = 0.9,
+                 const double beta2 = 0.999) :
+      finalLr(finalLr),
+      gamma(gamma),
+      epsilon(epsilon),
+      beta1(beta1),
+      beta2(beta2),
+      iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  //! Get the final (SGD) learning rate.
+  double FinalLr() const { return finalLr; }
+  //! Modify the final (SGD) learning rate.
+  double& FinalLr() { return finalLr; }
+
+  //! Get the convergence speed of the bound functions.
+  double Gamma() const { return finalLr; }
+  //! Modify the convergence speed of the bound functions.
+  double& Gamma() { return finalLr; }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AMSBoundUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AMSBoundUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent), first(true), initialStepSize(0)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+      vImproved.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AMSBound.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Convenience typedefs.
+      typedef typename MatType::elem_type ElemType;
+
+      // Save the initial step size.
+      if (first)
+      {
+        first = false;
+        initialStepSize = stepSize;
+      }
+
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // Decay the first and second moment running average coefficient.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
+      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma *
+          parent.iteration + 1));
+      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma *
+          parent.iteration));
+
+      // Element wise maximum of past and present squared gradients.
+      vImproved = arma::max(vImproved, v);
+
+      // Applies bounds on actual learning rate.
+      iterate -= arma::clamp((stepSize *
+          std::sqrt(biasCorrection2) / biasCorrection1) /
+          (arma::sqrt(vImproved) + parent.epsilon),  lower, upper) % m;
+    }
+
+   private:
+    // Instantiated parent object.
+    AMSBoundUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // Whether this is the first call of the Update method.
+    bool first;
+
+    // The initial (Adam) learning rate.
+    double initialStepSize;
+
+    // The optimal squared gradient value.
+    GradType vImproved;
+  };
+
+ private:
+  // The final (SGD) learning rate.
+  double finalLr;
+
+  // The convergence speed of the bound functions.
+  double gamma;
+
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The number of iterations.
+  size_t iteration;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ada_delta/ada_delta.hpp

@@ -66,6 +66,8 @@ class AdaDelta
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   AdaDelta(const double stepSize = 1.0,
            const size_t batchSize = 32,
@@ -74,23 +76,50 @@ class AdaDelta
            const size_t maxIterations = 100000,
            const double tolerance = 1e-5,
            const bool shuffle = true,
-           const bool resetPolicy = true);
+           const bool resetPolicy = true,
+           const bool exactObjective = false);
 
   /**
    * Optimize the given function using AdaDelta. The given starting point will
    * be modified to store the finishing point of the algorithm, and the final
-   * objective value is returned. The DecomposableFunctionType is checked for
+   * objective value is returned. The SeparableFunctionType is checked for
    * API consistency at compile time.
    *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -128,6 +157,11 @@ class AdaDelta
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/ada_delta/ada_delta_impl.hpp

@@ -26,7 +26,8 @@ inline AdaDelta::AdaDelta(const double stepSize,
                           const size_t maxIterations,
                           const double tolerance,
                           const bool shuffle,
-                          const bool resetPolicy) :
+                          const bool resetPolicy,
+                          const bool exactObjective) :
     optimizer(stepSize,
               batchSize,
               maxIterations,
@@ -34,7 +35,8 @@ inline AdaDelta::AdaDelta(const double stepSize,
               shuffle,
               AdaDeltaUpdate(rho, epsilon),
               NoDecay(),
-              resetPolicy)
+              resetPolicy,
+              exactObjective)
 { /* Nothing to do. */ }
 
 } // namespace ens

include/ensmallen_bits/ada_delta/ada_delta_update.hpp

@@ -51,49 +51,6 @@ class AdaDeltaUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process. In AdaDelta update policy, the mean squared
-   * and the delta mean squared gradient matrices are initialized to the zeros
-   * matrix with the same size as gradient matrix (see ens::SGD<>).
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    // Initialize empty matrices for mean sum of squares of parameter gradient.
-    meanSquaredGradient = arma::zeros<arma::mat>(rows, cols);
-    meanSquaredGradientDx = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for SGD. The AdaDelta update dynamically adapts over time using
-   * only first order information. Additionally, AdaDelta requires no manual
-   * tuning of a learning rate.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Accumulate gradient.
-    meanSquaredGradient *= rho;
-    meanSquaredGradient += (1 - rho) * (gradient % gradient);
-    arma::mat dx = arma::sqrt((meanSquaredGradientDx + epsilon) /
-        (meanSquaredGradient + epsilon)) % gradient;
-
-    // Accumulate updates.
-    meanSquaredGradientDx *= rho;
-    meanSquaredGradientDx += (1 - rho) * (dx % dx);
-
-    // Apply update.
-    iterate -= (stepSize * dx);
-  }
-
   //! Get the smoothing parameter.
   double Rho() const { return rho; }
   //! Modify the smoothing parameter.
@@ -104,18 +61,77 @@ class AdaDeltaUpdate
   //! Modify the value used to initialise the mean squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD optimizer method before the start
+     * of the iteration update process. In AdaDelta update policy, the mean
+     * squared and the delta mean squared gradient matrices are initialized to
+     * the zeros matrix with the same size as gradient matrix (see ens::SGD<>).
+     *
+     * @param parent AdaDeltaUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaDeltaUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      meanSquaredGradient.zeros(rows, cols);
+      meanSquaredGradientDx.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for SGD. The AdaDelta update dynamically adapts over time
+     * using only first order information. Additionally, AdaDelta requires no
+     * manual tuning of a learning rate.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Accumulate gradient.
+      meanSquaredGradient *= parent.rho;
+      meanSquaredGradient += (1 - parent.rho) * (gradient % gradient);
+      GradType dx = arma::sqrt((meanSquaredGradientDx + parent.epsilon) /
+          (meanSquaredGradient + parent.epsilon)) % gradient;
+
+      // Accumulate updates.
+      meanSquaredGradientDx *= parent.rho;
+      meanSquaredGradientDx += (1 - parent.rho) * (dx % dx);
+
+      // Apply update.
+      iterate -= (stepSize * dx);
+    }
+
+   private:
+    // The instantiated parent class.
+    AdaDeltaUpdate& parent;
+
+    // The mean squared gradient matrix.
+    GradType meanSquaredGradient;
+
+    // The delta mean squared gradient matrix.
+    GradType meanSquaredGradientDx;
+  };
+
  private:
   // The smoothing parameter.
   double rho;
 
   // The epsilon value used to initialise the mean squared gradient parameter.
   double epsilon;
-
-  // The mean squared gradient matrix.
-  arma::mat meanSquaredGradient;
-
-  // The delta mean squared gradient matrix.
-  arma::mat meanSquaredGradientDx;
 };
 
 } // namespace ens

include/ensmallen_bits/ada_grad/ada_grad.hpp

@@ -64,6 +64,8 @@ class AdaGrad
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   AdaGrad(const double stepSize = 0.01,
           const size_t batchSize = 32,
@@ -71,22 +73,49 @@ class AdaGrad
           const size_t maxIterations = 100000,
           const double tolerance = 1e-5,
           const bool shuffle = true,
-          const bool resetPolicy = true);
+          const bool resetPolicy = true,
+          const bool exactObjective = false);
 
   /**
    * Optimize the given function using AdaGrad. The given starting point will
    * be modified to store the finishing point of the algorithm, and the final
    * objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -119,6 +148,11 @@ class AdaGrad
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/ada_grad/ada_grad_impl.hpp

@@ -23,7 +23,8 @@ inline AdaGrad::AdaGrad(const double stepSize,
                         const size_t maxIterations,
                         const double tolerance,
                         const bool shuffle,
-                        const bool resetPolicy) :
+                        const bool resetPolicy,
+                        const bool exactObjective) :
     optimizer(stepSize,
               batchSize,
               maxIterations,
@@ -31,7 +32,8 @@ inline AdaGrad::AdaGrad(const double stepSize,
               shuffle,
               AdaGradUpdate(epsilon),
               NoDecay(),
-              resetPolicy)
+              resetPolicy,
+              exactObjective)
 { /* Nothing to do. */ }
 
 } // namespace ens

include/ensmallen_bits/ada_grad/ada_grad_update.hpp

@@ -49,49 +49,67 @@ class AdaGradUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process. In AdaGrad update policy, squared
-   * gradient matrix is initialized to the zeros matrix with the same size as
-   * gradient matrix (see ens::SGD<>).
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    // Initialize an empty matrix for sum of squares of parameter gradient.
-    squaredGradient = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for SGD. The AdaGrad update adapts the learning rate by
-   * performing larger updates for more sparse parameters and smaller updates
-   * for less sparse parameters .
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    squaredGradient += (gradient % gradient);
-    iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) + epsilon);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD optimizer before the start of the
+     * iteration update process. In AdaGrad update policy, squared gradient
+     * matrix is initialized to the zeros matrix with the same size as gradient
+     * matrix (see ens::SGD<>).
+     *
+     * @param parent Instantiated parent class.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaGradUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        squaredGradient(rows, cols)
+    {
+      // Initialize an empty matrix for sum of squares of parameter gradient.
+      squaredGradient.zeros();
+    }
+
+    /**
+     * Update step for SGD. The AdaGrad update adapts the learning rate by
+     * performing larger updates for more sparse parameters and smaller updates
+     * for less sparse parameters.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      squaredGradient += (gradient % gradient);
+      iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) +
+          parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent class.
+    AdaGradUpdate& parent;
+    // The squared gradient matrix.
+    GradType squaredGradient;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
-
-  // The squared gradient matrix.
-  arma::mat squaredGradient;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/adam.hpp

@@ -11,7 +11,7 @@
  * for first-order gradient-based optimization of stochastic objective
  * functions, based on adaptive estimates of lower-order moments. AdaMax is
  * simply a variant of Adam based on the infinity norm. AMSGrad is another
- * variant of Adam with guaranteed convergence. Nadam is another variant of 
+ * variant of Adam with guaranteed convergence. Nadam is another variant of
  * Adam based on NAG. NadaMax is a variant for Nadam based on Infinity form.
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -88,6 +88,8 @@ class AdamType
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   AdamType(const double stepSize = 0.001,
            const size_t batchSize = 32,
@@ -97,22 +99,49 @@ class AdamType
            const size_t maxIterations = 100000,
            const double tolerance = 1e-5,
            const bool shuffle = true,
-           const bool resetPolicy = true);
+           const bool resetPolicy = true,
+           const bool exactObjective = false);
 
   /**
    * Optimize the given function using Adam. The given starting point will be
    * modified to store the finishing point of the algorithm, and the final
    * objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.template Optimize<
+        SeparableFunctionType, MatType, GradType, CallbackTypes...>(
+        function, iterate, std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -155,6 +184,11 @@ class AdamType
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/adam/adam_impl.hpp

@@ -30,7 +30,8 @@ AdamType<UpdateRule>::AdamType(
     const size_t maxIterations,
     const double tolerance,
     const bool shuffle,
-    const bool resetPolicy) :
+    const bool resetPolicy,
+    const bool exactObjective) :
     optimizer(stepSize,
               batchSize,
               maxIterations,
@@ -38,7 +39,8 @@ AdamType<UpdateRule>::AdamType(
               shuffle,
               UpdateRule(epsilon, beta1, beta2),
               NoDecay(),
-              resetPolicy)
+              resetPolicy,
+              exactObjective)
 { /* Nothing to do. */ }
 
 } // namespace ens

include/ensmallen_bits/adam/adam_update.hpp

@@ -58,52 +58,6 @@ class AdamUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for Adam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    /**
-     * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
-     * following expression is an approximation of the following actual term;
-     * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
-     */
-    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-        m / (arma::sqrt(v) + epsilon);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -119,6 +73,82 @@ class AdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdamUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdamUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for Adam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      /**
+       * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
+       * following expression is an approximation of the following actual term;
+       * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
+       */
+      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+          m / (arma::sqrt(v) + parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent object.
+    AdamUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -129,14 +159,8 @@ class AdamUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat v;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/adamax_update.hpp

@@ -60,47 +60,6 @@ class AdaMaxUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    u = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for Adam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    // Update the exponentially weighted infinity norm.
-    u *= beta2;
-    u = arma::max(u, arma::abs(gradient));
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-
-    if (biasCorrection1 != 0)
-      iterate -= (stepSize / biasCorrection1 * m / (u + epsilon));
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -116,6 +75,74 @@ class AdaMaxUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdaMaxUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AdaMaxUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      u.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AdaMax.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      // Update the exponentially weighted infinity norm.
+      u *= parent.beta2;
+      u = arma::max(u, arma::abs(gradient));
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+
+      if (biasCorrection1 != 0)
+        iterate -= (stepSize / biasCorrection1 * m / (u + parent.epsilon));
+    }
+
+   private:
+    // Instantiated parent object.
+    AdaMaxUpdate& parent;
+    // The exponential moving average of gradient values.
+    GradType m;
+    // The exponentially weighted infinity norm.
+    GradType u;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -126,14 +153,8 @@ class AdaMaxUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponentially weighted infinity norm.
-  arma::mat u;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/amsgrad_update.hpp

@@ -53,51 +53,6 @@ class AMSGradUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-    vImproved = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for AMSGrad.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    // Element wise maximum of past and present squared gradients.
-    vImproved = arma::max(vImproved, v);
-
-    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-                m / (arma::sqrt(vImproved) + epsilon);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -113,6 +68,84 @@ class AMSGradUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated AMSGradUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(AMSGradUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+      vImproved.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for AMSGrad.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      // Element wise maximum of past and present squared gradients.
+      vImproved = arma::max(vImproved, v);
+
+      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                  m / (arma::sqrt(vImproved) + parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent AMSGradUpdate object.
+    AMSGradUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // The optimal squared gradient value.
+    GradType vImproved;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -123,17 +156,8 @@ class AMSGradUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat v;
-
-  // The optimal sqaured gradient value.
-  arma::mat vImproved;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/nadam_update.hpp

@@ -51,78 +51,16 @@ class NadamUpdate
       beta1(beta1),
       beta2(beta2),
       scheduleDecay(scheduleDecay),
-      iteration(0),
-      cumBeta1(1)
+      iteration(0)
   {
     // Nothing to do.
   }
 
-  /**
-   * The Initialize() method is called by the optimizer before the start of the
-   * iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for Nadam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * gradient % gradient;
-
-    double beta1T = beta1 * (1 - (0.5 *
-        std::pow(0.96, iteration * scheduleDecay)));
-
-    double beta1T1 = beta1 * (1 - (0.5 *
-        std::pow(0.96, (iteration + 1) * scheduleDecay)));
-
-    cumBeta1 *= beta1T;
-
-    const double biasCorrection1 = 1.0 - cumBeta1;
-
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
-
-    /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
-     * as arma::sqrt(v) + epsilon
-     */
-    iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
-        + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2))
-        / (arma::sqrt(v) + epsilon);
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
-  //! Get the value of the cumulative product of decay coefficients
-  double CumBeta1() const { return cumBeta1; }
-  //! Modify the value of the cumulative product of decay coefficients
-  double& CumBeta1() { return cumBeta1; }
-
   //! Get the smoothing parameter.
   double Beta1() const { return beta1; }
   //! Modify the smoothing parameter.
@@ -138,6 +76,95 @@ class NadamUpdate
   //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the optimizer before the start of the
+     * iteration update process.
+     *
+     * @param parent Instantiated NadamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(NadamUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        cumBeta1(1)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for Nadam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * gradient % gradient;
+
+      double beta1T = parent.beta1 * (1 - (0.5 *
+          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+
+      double beta1T1 = parent.beta1 * (1 - (0.5 *
+          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+
+      cumBeta1 *= beta1T;
+
+      const double biasCorrection1 = 1.0 - cumBeta1;
+
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
+
+      /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
+       * as arma::sqrt(v) + epsilon
+       */
+      iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
+          + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2))
+          / (arma::sqrt(v) + parent.epsilon);
+    }
+
+   private:
+    // Instantiated parent object.
+    NadamUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // The cumulative product of decay coefficients.
+    double cumBeta1;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -148,20 +175,11 @@ class NadamUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat v;
-
-  // The decay parameter for decay coefficients
+  // The decay parameter for decay coefficients.
   double scheduleDecay;
 
   // The number of iterations.
-  double iteration;
-
-  // The cumulative product of decay coefficients
-  double cumBeta1;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/nadamax_update.hpp

@@ -51,74 +51,16 @@ class NadaMaxUpdate
       beta1(beta1),
       beta2(beta2),
       scheduleDecay(scheduleDecay),
-      cumBeta1(1),
       iteration(0)
   {
     // Nothing to do.
   }
 
-  /**
-   * The Initialize() method is called by the optimizer before the start of the
-   * iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    u = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for NadaMax.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    u = arma::max(u * beta2, arma::abs(gradient));
-
-    double beta1T = beta1 * (1 - (0.5 *
-        std::pow(0.96, iteration * scheduleDecay)));
-
-    double beta1T1 = beta1 * (1 - (0.5 *
-        std::pow(0.96, (iteration + 1) * scheduleDecay)));
-
-    cumBeta1 *= beta1T;
-
-    const double biasCorrection1 = 1.0 - cumBeta1;
-
-    const double biasCorrection2 = 1.0 - (cumBeta1 * beta1T1);
-
-    if ((biasCorrection1 != 0) && (biasCorrection2 != 0))
-    {
-       iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
-           + (beta1T1 / biasCorrection2) * m)) / (u + epsilon);
-    }
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
-  //! Get the value of the cumulative product of decay coefficients
-  double CumBeta1() const { return cumBeta1; }
-  //! Modify the value of the cumulative product of decay coefficients
-  double& CumBeta1() { return cumBeta1; }
-
   //! Get the smoothing parameter.
   double Beta1() const { return beta1; }
   //! Modify the smoothing parameter.
@@ -134,6 +76,90 @@ class NadaMaxUpdate
   //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor method is called by the optimizer before the start of
+     * the iteration update process.
+     *
+     * @param parent Instantiated NadaMaxUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(NadaMaxUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent),
+        cumBeta1(1)
+    {
+      m.zeros(rows, cols);
+      u.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for NadaMax.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      u = arma::max(u * parent.beta2, arma::abs(gradient));
+
+      double beta1T = parent.beta1 * (1 - (0.5 *
+          std::pow(0.96, parent.iteration * parent.scheduleDecay)));
+
+      double beta1T1 = parent.beta1 * (1 - (0.5 *
+          std::pow(0.96, (parent.iteration + 1) * parent.scheduleDecay)));
+
+      cumBeta1 *= beta1T;
+
+      const double biasCorrection1 = 1.0 - cumBeta1;
+
+      const double biasCorrection2 = 1.0 - (cumBeta1 * beta1T1);
+
+      if ((biasCorrection1 != 0) && (biasCorrection2 != 0))
+      {
+         iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
+             + (beta1T1 / biasCorrection2) * m)) / (u + parent.epsilon);
+      }
+    }
+
+   private:
+    // Instantiated parent object.
+    NadaMaxUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponentially weighted infinity norm.
+    GradType u;
+
+    // The cumulative product of decay coefficients.
+    double cumBeta1;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -144,20 +170,11 @@ class NadaMaxUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponentially weighted infinity norm.
-  arma::mat u;
-
-  // The decay parameter for decay coefficients
+  // The decay parameter for decay coefficients.
   double scheduleDecay;
 
-  // The cumulative product of decay coefficients
-  double cumBeta1;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/adam/optimisticadam_update.hpp

@@ -2,7 +2,7 @@
  * @file optimisticadam_update.hpp
  * @author Moksh Jain
  *
- * OptmisticAdam optimizer. Implements Optimistic Adam, an algorithm which 
+ * OptmisticAdam optimizer. Implements Optimistic Adam, an algorithm which
  * uses Optimistic Mirror Descent with the Adam optimizer.
  *
  * ensmallen is free software; you may redistribute it and/or modify it under
@@ -16,18 +16,18 @@
 namespace ens {
 
 /**
- * OptimisticAdam is an optimizer which implements the Optimistic Adam 
+ * OptimisticAdam is an optimizer which implements the Optimistic Adam
  * algorithm which uses Optmistic Mirror Descent with the Adam Optimizer.
  * It addresses the problem of limit cycling while training GANs. It uses
- * OMD to achieve faster regret rates in solving the zero sum game of 
- * training a GAN. It consistently achieves a smaller KL divergnce with 
+ * OMD to achieve faster regret rates in solving the zero sum game of
+ * training a GAN. It consistently achieves a smaller KL divergnce with
  * respect to the true underlying data distribution.
  *
  * For more information, see the following.
  *
  * @code
  * @article{
- *   author  = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis, 
+ *   author  = {Constantinos Daskalakis, Andrew Ilyas, Vasilis Syrgkanis,
  *              Haoyang Zeng},
  *   title   = {Training GANs with Optimism},
  *   year    = {2017},
@@ -57,51 +57,6 @@ class OptimisticAdamUpdate
     // Nothing to do.
   }
 
-  /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    m = arma::zeros<arma::mat>(rows, cols);
-    v = arma::zeros<arma::mat>(rows, cols);
-    g = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for OptimisticAdam.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    v *= beta2;
-    v += (1 - beta2) * arma::square(gradient);
-
-    arma::mat mCorrected = m / (1.0 - std::pow(beta1, iteration));
-    arma::mat vCorrected = v / (1.0 - std::pow(beta2, iteration));
-
-    arma::mat update = mCorrected / (arma::sqrt(vCorrected) + epsilon);
-
-    iterate -= (2 * stepSize * update - stepSize * g);
-
-    g = std::move(update);
-  }
-
   //! Get the value used to initialize the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialize the squared gradient parameter.
@@ -117,6 +72,85 @@ class OptimisticAdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent Instantiated OptimisticAdamUpdate parent object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(OptimisticAdamUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      m.zeros(rows, cols);
+      v.zeros(rows, cols);
+      g.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for OptimisticAdam.
+     *
+     * @param iterate Parameters that minimize the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      m *= parent.beta1;
+      m += (1 - parent.beta1) * gradient;
+
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * arma::square(gradient);
+
+      GradType mCorrected = m / (1.0 - std::pow(parent.beta1,
+          parent.iteration));
+      GradType vCorrected = v / (1.0 - std::pow(parent.beta2,
+          parent.iteration));
+
+      GradType update = mCorrected /
+          (arma::sqrt(vCorrected) + parent.epsilon);
+
+      iterate -= (2 * stepSize * update - stepSize * g);
+
+      g = std::move(update);
+    }
+
+   private:
+    // Instantiated parent object.
+    OptimisticAdamUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType m;
+
+    // The exponential moving average of squared gradient values.
+    GradType v;
+
+    // The previous update.
+    GradType g;
+  };
+
  private:
   // The epsilon value used to initialize the squared gradient parameter.
   double epsilon;
@@ -127,16 +161,8 @@ class OptimisticAdamUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat m;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat v;
-  // The previous update.
-  arma::mat g;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp

@@ -55,12 +55,34 @@ class AugLagrangian
    *
    * @tparam LagrangianFunctionType Function which can be optimized by this
    *     class.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function The function to optimize.
    * @param coordinates Output matrix to store the optimized coordinates in.
+   * @param callbacks Callback functions.
    */
-  template<typename LagrangianFunctionType>
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  Optimize(LagrangianFunctionType& function,
+           MatType& coordinates,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
   bool Optimize(LagrangianFunctionType& function,
-                arma::mat& coordinates);
+                MatType& coordinates,
+                CallbackTypes&&... callbacks)
+  {
+    return Optimize<LagrangianFunctionType, MatType, MatType,
+        CallbackTypes...>(function, coordinates,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   /**
    * Optimize the function, giving initial estimates for the Lagrange
@@ -69,17 +91,41 @@ class AugLagrangian
    *
    * @tparam LagrangianFunctionType Function which can be optimized by this
    *      class.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function The function to optimize.
    * @param coordinates Output matrix to store the optimized coordinates in.
    * @param initLambda Vector of initial Lagrange multipliers.  Should have
    *     length equal to the number of constraints.
    * @param initSigma Initial penalty parameter.
+   * @param callbacks Callback functions.
    */
-  template<typename LagrangianFunctionType>
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  Optimize(LagrangianFunctionType& function,
+           MatType& coordinates,
+           const arma::vec& initLambda,
+           const double initSigma,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
   bool Optimize(LagrangianFunctionType& function,
-                arma::mat& coordinates,
+                MatType& coordinates,
                 const arma::vec& initLambda,
-                const double initSigma);
+                const double initSigma,
+                CallbackTypes&&... callbacks)
+  {
+    return Optimize<LagrangianFunctionType, MatType, MatType,
+        CallbackTypes...>(function, coordinates, initLambda, initSigma,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the L-BFGS object used for the actual optimization.
   const L_BFGS& LBFGS() const { return lbfgs; }
@@ -124,8 +170,12 @@ class AugLagrangian
   //! The L-BFGS optimizer that we will use.
   L_BFGS lbfgs;
 
+  //! Controls early termination of the optimization process.
+  bool terminate;
+
   //! Lagrange multipliers.
   arma::vec lambda;
+
   //! Penalty parameter.
   double sigma;
 
@@ -133,9 +183,27 @@ class AugLagrangian
    * Internal optimization function: given an initialized AugLagrangianFunction,
    * perform the optimization itself.
    */
-  template<typename LagrangianFunctionType>
-  bool Optimize(AugLagrangianFunction<LagrangianFunctionType>& augfunc,
-                arma::mat& coordinates);
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+  Optimize(AugLagrangianFunction<LagrangianFunctionType>& augfunc,
+           MatType& coordinates,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename LagrangianFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  bool Optimize(AugLagrangianFunction<LagrangianFunctionType>& function,
+                MatType& coordinates,
+                CallbackTypes&&... callbacks)
+  {
+    return Optimize<LagrangianFunctionType, MatType, MatType,
+        CallbackTypes...>(function, coordinates,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 };
 
 } // namespace ens

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_function.hpp

@@ -63,7 +63,8 @@ class AugLagrangianFunction
    * @param coordinates Coordinates to evaluate function at.
    * @return Objective function.
    */
-  double Evaluate(const arma::mat& coordinates) const;
+  template<typename MatType>
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const;
 
   /**
    * Evaluate the gradient of the Augmented Lagrangian function.
@@ -71,7 +72,8 @@ class AugLagrangianFunction
    * @param coordinates Coordinates to evaluate gradient at.
    * @param gradient Matrix to store gradient into.
    */
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const;
+  template<typename MatType, typename GradType>
+  void Gradient(const MatType& coordinates, GradType& gradient) const;
 
   /**
    * Get the initial point of the optimization (supplied by the
@@ -79,7 +81,8 @@ class AugLagrangianFunction
    *
    * @return Initial point.
    */
-  const arma::mat& GetInitialPoint() const;
+  template<typename MatType = arma::mat>
+  const MatType& GetInitialPoint() const;
 
   //! Get the Lagrange multipliers.
   const arma::vec& Lambda() const { return lambda; }

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_function_impl.hpp

@@ -46,19 +46,22 @@ AugLagrangianFunction<LagrangianFunction>::AugLagrangianFunction(
 
 // Evaluate the AugLagrangianFunction at the given coordinates.
 template<typename LagrangianFunction>
-double AugLagrangianFunction<LagrangianFunction>::Evaluate(
-    const arma::mat& coordinates) const
+template<typename MatType>
+typename MatType::elem_type AugLagrangianFunction<LagrangianFunction>::Evaluate(
+    const MatType& coordinates) const
 {
   // The augmented Lagrangian is evaluated as
   //    f(x) + {-lambda_i * c_i(x) + (sigma / 2) c_i(x)^2} for all constraints
 
+  typedef typename MatType::elem_type ElemType;
+
   // First get the function's objective value.
-  double objective = function.Evaluate(coordinates);
+  ElemType objective = function.Evaluate(coordinates);
 
   // Now loop for each constraint.
   for (size_t i = 0; i < function.NumConstraints(); ++i)
   {
-    double constraint = function.EvaluateConstraint(i, coordinates);
+    ElemType constraint = function.EvaluateConstraint(i, coordinates);
 
     objective += (-lambda[i] * constraint) +
         sigma * std::pow(constraint, 2) / 2;
@@ -69,22 +72,23 @@ double AugLagrangianFunction<LagrangianFunction>::Evaluate(
 
 // Evaluate the gradient of the AugLagrangianFunction at the given coordinates.
 template<typename LagrangianFunction>
+template<typename MatType, typename GradType>
 void AugLagrangianFunction<LagrangianFunction>::Gradient(
-    const arma::mat& coordinates,
-    arma::mat& gradient) const
+    const MatType& coordinates,
+    GradType& gradient) const
 {
   // The augmented Lagrangian's gradient is evaluted as
   // f'(x) + {(-lambda_i + sigma * c_i(x)) * c'_i(x)} for all constraints
   gradient.zeros();
   function.Gradient(coordinates, gradient);
 
-  arma::mat constraintGradient; // Temporary for constraint gradients.
+  GradType constraintGradient; // Temporary for constraint gradients.
   for (size_t i = 0; i < function.NumConstraints(); i++)
   {
     function.GradientConstraint(i, coordinates, constraintGradient);
 
     // Now calculate scaling factor and add to existing gradient.
-    arma::mat tmpGradient;
+    GradType tmpGradient;
     tmpGradient = (-lambda[i] + sigma *
         function.EvaluateConstraint(i, coordinates)) * constraintGradient;
     gradient += tmpGradient;
@@ -93,10 +97,11 @@ void AugLagrangianFunction<LagrangianFunction>::Gradient(
 
 // Get the initial point.
 template<typename LagrangianFunction>
-const arma::mat& AugLagrangianFunction<LagrangianFunction>::GetInitialPoint()
+template<typename MatType>
+const MatType& AugLagrangianFunction<LagrangianFunction>::GetInitialPoint()
     const
 {
-  return function.GetInitialPoint();
+  return function.template GetInitialPoint<MatType>();
 }
 
 } // namespace ens

include/ensmallen_bits/aug_lagrangian/aug_lagrangian_impl.hpp

@@ -26,15 +26,22 @@ inline AugLagrangian::AugLagrangian(const size_t maxIterations,
     maxIterations(maxIterations),
     penaltyThresholdFactor(penaltyThresholdFactor),
     sigmaUpdateFactor(sigmaUpdateFactor),
-    lbfgs(lbfgs)
+    lbfgs(lbfgs),
+    terminate(false),
+    sigma(0.0)
 {
 }
 
-template<typename LagrangianFunctionType>
-bool AugLagrangian::Optimize(LagrangianFunctionType& function,
-                             arma::mat& coordinates,
-                             const arma::vec& initLambda,
-                             const double initSigma)
+template<typename LagrangianFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+AugLagrangian::Optimize(LagrangianFunctionType& function,
+                        MatType& coordinates,
+                        const arma::vec& initLambda,
+                        const double initSigma,
+                        CallbackTypes&&... callbacks)
 {
   lambda = initLambda;
   sigma = initSigma;
@@ -42,46 +49,79 @@ bool AugLagrangian::Optimize(LagrangianFunctionType& function,
   AugLagrangianFunction<LagrangianFunctionType> augfunc(function,
       lambda, sigma);
 
-  return Optimize(augfunc, coordinates);
+  return Optimize(augfunc, coordinates, callbacks...);
 }
 
-template<typename LagrangianFunctionType>
-bool AugLagrangian::Optimize(LagrangianFunctionType& function,
-                             arma::mat& coordinates)
+template<typename LagrangianFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+AugLagrangian::Optimize(LagrangianFunctionType& function,
+                        MatType& coordinates,
+                        CallbackTypes&&... callbacks)
 {
   // If the user did not specify the right size for sigma and lambda, we will
   // use defaults.
   if (!lambda.is_empty())
   {
-    AugLagrangianFunction<LagrangianFunctionType> augfunc(function, lambda, sigma);
-    return Optimize(augfunc, coordinates);
+    AugLagrangianFunction<LagrangianFunctionType> augfunc(function, lambda,
+        sigma);
+    return Optimize(augfunc, coordinates, callbacks...);
   }
   else
   {
     AugLagrangianFunction<LagrangianFunctionType> augfunc(function);
-    return Optimize(augfunc, coordinates);
+    return Optimize(augfunc, coordinates, callbacks...);
   }
 }
 
-template<typename LagrangianFunctionType>
-bool AugLagrangian::Optimize(
+template<typename LagrangianFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value, bool>::type
+AugLagrangian::Optimize(
     AugLagrangianFunction<LagrangianFunctionType>& augfunc,
-    arma::mat& coordinates)
+    MatType& coordinatesIn,
+    CallbackTypes&&... callbacks)
 {
-  traits::CheckConstrainedFunctionTypeAPI<LagrangianFunctionType>();
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  BaseMatType& coordinates = (BaseMatType&) coordinatesIn;
+
+  // Check that the types satisfy our needs.
+  traits::CheckConstrainedFunctionTypeAPI<LagrangianFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
 
   LagrangianFunctionType& function = augfunc.Function();
 
   // Ensure that we update lambda immediately.
-  double penaltyThreshold = DBL_MAX;
+  ElemType penaltyThreshold = std::numeric_limits<ElemType>::max();
 
   // Track the last objective to compare for convergence.
-  double lastObjective = function.Evaluate(coordinates);
+  ElemType lastObjective = function.Evaluate(coordinates);
+
+  // Convergence tolerance---depends on the epsilon of the type we are using for
+  // optimization.
+  ElemType tolerance = 1e3 * std::numeric_limits<ElemType>::epsilon();
 
   // Then, calculate the current penalty.
-  double penalty = 0;
+  ElemType penalty = 0;
   for (size_t i = 0; i < function.NumConstraints(); i++)
-    penalty += std::pow(function.EvaluateConstraint(i, coordinates), 2);
+  {
+    const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates), 2);
+    Callback::EvaluateConstraint(*this, function, coordinates, i, p,
+          callbacks...);
+
+    penalty += p;
+  }
 
   Info << "Penalty is " << penalty << " (threshold " << penaltyThreshold
       << ")." << std::endl;
@@ -89,36 +129,51 @@ bool AugLagrangian::Optimize(
   // The odd comparison allows user to pass maxIterations = 0 (i.e. no limit on
   // number of iterations).
   size_t it;
-  for (it = 0; it != (maxIterations - 1); it++)
+  terminate |= Callback::BeginOptimization(*this, function, coordinates,
+      callbacks...);
+  for (it = 0; it != (maxIterations - 1) && !terminate; it++)
   {
     Info << "AugLagrangian on iteration " << it
         << ", starting with objective "  << lastObjective << "." << std::endl;
 
-    if (!lbfgs.Optimize(augfunc, coordinates))
+    if (!lbfgs.Optimize(augfunc, coordinates, callbacks...))
       Info << "L-BFGS reported an error during optimization."
           << std::endl;
+    Info << "Done with L-BFGS: " << coordinates << "\n";
+
+    const ElemType objective = function.Evaluate(coordinates);
+
+    Callback::Evaluate(*this, function, coordinates, objective,
+        callbacks...);
 
     // Check if we are done with the entire optimization (the threshold we are
     // comparing with is arbitrary).
-    if (std::abs(lastObjective - function.Evaluate(coordinates)) < 1e-10 &&
+    if (std::abs(lastObjective - objective) < tolerance &&
         augfunc.Sigma() > 500000)
     {
       lambda = std::move(augfunc.Lambda());
       sigma = augfunc.Sigma();
+
+      Callback::EndOptimization(*this, function, coordinates, callbacks...);
       return true;
     }
 
-    lastObjective = function.Evaluate(coordinates);
+    lastObjective = objective;
 
     // Assuming that the optimization has converged to a new set of coordinates,
     // we now update either lambda or sigma.  We update sigma if the penalty
     // term is too high, and we update lambda otherwise.
 
     // First, calculate the current penalty.
-    double penalty = 0;
+    ElemType penalty = 0;
     for (size_t i = 0; i < function.NumConstraints(); i++)
     {
-      penalty += std::pow(function.EvaluateConstraint(i, coordinates), 2);
+      const ElemType p = std::pow(function.EvaluateConstraint(i, coordinates),
+          2);
+      Callback::EvaluateConstraint(*this, function, coordinates, i, p,
+          callbacks...);
+
+      penalty += p;
     }
 
     Info << "Penalty is " << penalty << " (threshold "
@@ -129,8 +184,13 @@ bool AugLagrangian::Optimize(
       // We use the update: lambda_{k + 1} = lambda_k - sigma * c(coordinates),
       // but we have to write a loop to do this for each constraint.
       for (size_t i = 0; i < function.NumConstraints(); i++)
-        augfunc.Lambda()[i] -= augfunc.Sigma() *
-            function.EvaluateConstraint(i, coordinates);
+      {
+        const ElemType p = function.EvaluateConstraint(i, coordinates);
+        Callback::EvaluateConstraint(*this, function, coordinates, i, p,
+          callbacks...);
+
+        augfunc.Lambda()[i] -= augfunc.Sigma() * p;
+      }
 
       // We also update the penalty threshold to be a factor of the current
       // penalty.
@@ -142,9 +202,20 @@ bool AugLagrangian::Optimize(
       // We multiply sigma by a constant value.
       augfunc.Sigma() *= sigmaUpdateFactor;
       Info << "Updated sigma to " << augfunc.Sigma() << "." << std::endl;
+      if (augfunc.Sigma() >= std::numeric_limits<ElemType>::max() / 2.0)
+      {
+        Warn << "AugLagrangian::Optimize(): sigma too large for element type; "
+            << "terminating." << std::endl;
+        Callback::EndOptimization(*this, function, coordinates, callbacks...);
+        return false;
+      }
     }
+
+    terminate |= Callback::StepTaken(*this, function, coordinates,
+        callbacks...);
   }
 
+  Callback::EndOptimization(*this, function, coordinates, callbacks...);
   return false;
 }
 

include/ensmallen_bits/bigbatch_sgd/adaptive_stepsize.hpp

@@ -48,69 +48,12 @@ class AdaptiveStepsize
    * @param searchParameter The backtracking search parameter for each
    *        iteration.
    */
-  AdaptiveStepsize(const double backtrackStepSize = 0.1,
+  AdaptiveStepsize(const double backtrackStepSize = 0.5,
                    const double searchParameter = 0.1) :
       backtrackStepSize(backtrackStepSize),
       searchParameter(searchParameter)
   { /* Nothing to do here. */ }
 
-  /**
-   * This function is called in each iteration.
-   *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
-   * @param function Function to be optimized (minimized).
-   * @param stepSize Step size to be used for the given iteration.
-   * @param iterate Parameters that minimize the function.
-   * @param gradient The gradient matrix.
-   * @param gradientNorm The gradient norm to be used for the given iteration.
-   * @param offset The batch offset to be used for the given iteration.
-   * @param batchSize Batch size to be used for the given iteration.
-   * @param backtrackingBatchSize Backtracking batch size to be used for the
-   *        given iteration.
-   * @param reset Reset the step size decay parameter.
-   */
-  template<typename DecomposableFunctionType>
-  void Update(DecomposableFunctionType& function,
-              double& stepSize,
-              arma::mat& iterate,
-              const arma::mat& gradient,
-              const double gradientNorm,
-              const double sampleVariance,
-              const size_t offset,
-              const size_t batchSize,
-              const size_t backtrackingBatchSize,
-              const bool /* reset */)
-  {
-    Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
-        backtrackingBatchSize);
-
-    // Update the iterate.
-    iterate -= stepSize * gradient;
-
-    // TODO: Develop an absolute strategy to deal with stepSizeDecay updates in
-    // case we arrive at local minima. See #1469 for more details.
-    double stepSizeDecay = 0;
-    if (gradientNorm && sampleVariance && batchSize)
-    {
-      if (batchSize < function.NumFunctions())
-      {
-        stepSizeDecay = (1 - (1 / ((double) batchSize - 1) * sampleVariance) /
-            (batchSize * gradientNorm)) / batchSize;
-      }
-      else
-      {
-        stepSizeDecay = 1 / function.NumFunctions();
-      }
-    }
-
-    // Stepsize smoothing.
-    stepSize *= (1 - ((double) batchSize / function.NumFunctions()));
-    stepSize += stepSizeDecay * ((double) batchSize / function.NumFunctions());
-
-    Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
-        backtrackingBatchSize);
-  }
-
   //! Get the backtracking step size.
   double BacktrackStepSize() const { return backtrackStepSize; }
   //! Modify the backtracking step size.
@@ -121,48 +64,177 @@ class AdaptiveStepsize
   //! Modify the search parameter.
   double& SearchParameter() { return searchParameter; }
 
- private:
-  /**
-   * Definition of the backtracking line search algorithm based on the
-   * Armijo–Goldstein condition to determine the maximum amount to move along
-   * the given search direction.
-   *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
-   * @param function Function to be optimized (minimized).
-   * @param stepSize Step size to be used for the given iteration.
-   * @param iterate Parameters that minimize the function.
-   * @param gradient The gradient matrix.
-   * @param gradientNorm The gradient norm to be used for the given iteration.
-   * @param offset The batch offset to be used for the given iteration.
-   * @param backtrackingBatchSize The backtracking batch size.
-   */
-  template<typename DecomposableFunctionType>
-  void Backtracking(DecomposableFunctionType& function,
-                    double& stepSize,
-                    const arma::mat& iterate,
-                    const arma::mat& gradient,
-                    const double gradientNorm,
-                    const size_t offset,
-                    const size_t backtrackingBatchSize)
+
+  template<typename MatType>
+  class Policy
   {
-    double overallObjective = function.Evaluate(iterate, offset,
-        backtrackingBatchSize);
+   public:
+    // Create the instantiated object.
+    Policy(AdaptiveStepsize& parent) : parent(parent) { }
 
-    arma::mat iterateUpdate = iterate - (stepSize * gradient);
-    double overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
-        backtrackingBatchSize);
-
-    while (overallObjectiveUpdate >
-        (overallObjective + searchParameter * stepSize * gradientNorm))
+    /**
+     * This function is called in each iteration.
+     *
+     * @tparam SeparableFunctionType Type of the function to be optimized.
+     * @param function Function to be optimized (minimized).
+     * @param stepSize Step size to be used for the given iteration.
+     * @param iterate Parameters that minimize the function.
+     * @param gradient The gradient matrix.
+     * @param gradientNorm The gradient norm to be used for the given iteration.
+     * @param offset The batch offset to be used for the given iteration.
+     * @param batchSize Batch size to be used for the given iteration.
+     * @param backtrackingBatchSize Backtracking batch size to be used for the
+     *        given iteration.
+     * @param reset Reset the step size decay parameter.
+     */
+    template<typename SeparableFunctionType,
+             typename GradType>
+    void Update(SeparableFunctionType& function,
+                double& stepSize,
+                MatType& iterate,
+                GradType& gradient,
+                double& gradientNorm,
+                double& sampleVariance,
+                const size_t offset,
+                const size_t batchSize,
+                const size_t backtrackingBatchSize,
+                const bool /* reset */)
     {
-      stepSize *= backtrackStepSize;
+      Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
+          backtrackingBatchSize);
 
-      iterateUpdate = iterate - (stepSize * gradient);
-      overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
+      // Update the iterate.
+      iterate -= stepSize * gradient;
+
+      // Update Gradient & calculate curvature of quadratic approximation.
+      GradType functionGradient(iterate.n_rows, iterate.n_cols);
+      GradType gradPrevIterate(iterate.n_rows, iterate.n_cols);
+      GradType functionGradientPrev(iterate.n_rows, iterate.n_cols);
+
+      double vB = 0;
+      GradType delta0, delta1;
+
+      // Initialize previous iterate, if not already initialized.
+      if (iteratePrev.is_empty())
+      {
+        iteratePrev.zeros(iterate.n_rows, iterate.n_cols);
+      }
+
+      // Compute the stochastic gradient estimation.
+      function.Gradient(iterate, offset, gradient, 1);
+      function.Gradient(iteratePrev, offset, gradPrevIterate, 1);
+
+      delta1 = gradient;
+
+      for (size_t j = 1, k = 1; j < backtrackingBatchSize; ++j, ++k)
+      {
+        function.Gradient(iterate, offset + j, functionGradient, 1);
+        delta0 = delta1 + (functionGradient - delta1) / k;
+
+        // Compute sample variance.
+        vB += arma::norm(functionGradient - delta1, 2.0) *
+            arma::norm(functionGradient - delta0, 2.0);
+
+        delta1 = delta0;
+        gradient += functionGradient;
+
+        // Used for curvature calculation.
+        function.Gradient(iteratePrev, offset + j, functionGradientPrev, 1);
+        gradPrevIterate += functionGradientPrev;
+      }
+
+      // Update sample variance & norm of the gradient.
+      sampleVariance = vB;
+      gradientNorm = std::pow(arma::norm(gradient / backtrackingBatchSize, 2),
+          2.0);
+
+      // Compute curvature.
+      double v = arma::trace(arma::trans(iterate - iteratePrev) *
+          (gradient - gradPrevIterate)) /
+          std::pow(arma::norm(iterate - iteratePrev, 2), 2.0);
+
+      // Update previous iterate.
+      iteratePrev = iterate;
+
+      // TODO: Develop an absolute strategy to deal with stepSizeDecay updates
+      // in case we arrive at local minima. See #1469 for more details.
+      double stepSizeDecay = 0;
+      if (gradientNorm && sampleVariance && batchSize)
+      {
+        if (batchSize < function.NumFunctions())
+        {
+          stepSizeDecay = (1 - (1 / ((double) batchSize - 1) * sampleVariance) /
+              (batchSize * gradientNorm)) / v;
+        }
+        else
+        {
+          stepSizeDecay = 1 / v;
+        }
+      }
+
+      // Stepsize smoothing.
+      stepSize *= (1 - ((double) batchSize / function.NumFunctions()));
+      stepSize += stepSizeDecay * ((double) batchSize /
+          function.NumFunctions());
+
+      Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
           backtrackingBatchSize);
     }
-  }
 
+    /**
+     * Definition of the backtracking line search algorithm based on the
+     * Armijo–Goldstein condition to determine the maximum amount to move along
+     * the given search direction.
+     *
+     * @tparam SeparableFunctionType Type of the function to be optimized.
+     * @param function Function to be optimized (minimized).
+     * @param stepSize Step size to be used for the given iteration.
+     * @param iterate Parameters that minimize the function.
+     * @param gradient The gradient matrix.
+     * @param gradientNorm The gradient norm to be used for the given iteration.
+     * @param offset The batch offset to be used for the given iteration.
+     * @param backtrackingBatchSize The backtracking batch size.
+     */
+    template<typename SeparableFunctionType,
+             typename GradType>
+    void Backtracking(SeparableFunctionType& function,
+                      double& stepSize,
+                      const MatType& iterate,
+                      const GradType& gradient,
+                      const double gradientNorm,
+                      const size_t offset,
+                      const size_t backtrackingBatchSize)
+    {
+      typedef typename MatType::elem_type ElemType;
+
+      ElemType overallObjective = function.Evaluate(iterate,
+          offset, backtrackingBatchSize);
+
+      MatType iterateUpdate = iterate - (stepSize * gradient);
+      ElemType overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
+          backtrackingBatchSize);
+
+      while (overallObjectiveUpdate >
+          (overallObjective - parent.searchParameter * stepSize *
+           gradientNorm))
+      {
+        stepSize *= parent.backtrackStepSize;
+
+        iterateUpdate = iterate - (stepSize * gradient);
+        overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
+            backtrackingBatchSize);
+      }
+    }
+
+   private:
+    //! Reference to parent.
+    AdaptiveStepsize& parent;
+
+    //! Last function parameters value.
+    MatType iteratePrev;
+  };
+
+ private:
   //! The backtracking step size for each iteration.
   double backtrackStepSize;
 

include/ensmallen_bits/bigbatch_sgd/backtracking_line_search.hpp

@@ -51,53 +51,74 @@ class BacktrackingLineSearch
       searchParameter(searchParameter)
   { /* Nothing to do here. */ }
 
-  /**
-   * This function is called in each iteration.
-   *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
-   * @param function Function to be optimized (minimized).
-   * @param stepSize Step size to be used for the given iteration.
-   * @param iterate Parameters that minimize the function.
-   * @param gradient The gradient matrix.
-   * @param gradientNorm The gradient norm to be used for the given iteration.
-   * @param offset The batch offset to be used for the given iteration.
-   * @param batchSize Batch size to be used for the given iteration.
-   * @param backtrackingBatchSize Backtracking batch size to be used for the
-   *        given iteration.
-   * @param reset Reset the step size decay parameter.
-   */
-  template<typename DecomposableFunctionType>
-  void Update(DecomposableFunctionType& function,
-              double& stepSize,
-              arma::mat& iterate,
-              const arma::mat& gradient,
-              const double gradientNorm,
-              const double /* sampleVariance */,
-              const size_t offset,
-              const size_t /* batchSize */,
-              const size_t backtrackingBatchSize,
-              const bool reset)
+  //! Get the search parameter.
+  double SearchParameter() const { return searchParameter; }
+  //! Modify the search parameter.
+  double& SearchParameter() { return searchParameter; }
+
+  template<typename MatType>
+  class Policy
   {
-    if (reset)
-      stepSize *= 2;
+   public:
+    // Instantiate the policy with the given parent.
+    Policy(BacktrackingLineSearch& parent) : parent(parent) { }
 
-    double overallObjective = function.Evaluate(iterate, offset,
-        backtrackingBatchSize);
-
-    arma::mat iterateUpdate = iterate - (stepSize * gradient);
-    double overallObjectiveUpdate = function.Evaluate(iterateUpdate,
-        offset, backtrackingBatchSize);
-
-    while (overallObjectiveUpdate >
-        (overallObjective + searchParameter * stepSize * gradientNorm))
+    /**
+     * This function is called in each iteration.
+     *
+     * @tparam SeparableFunctionType Type of the function to be optimized.
+     * @param function Function to be optimized (minimized).
+     * @param stepSize Step size to be used for the given iteration.
+     * @param iterate Parameters that minimize the function.
+     * @param gradient The gradient matrix.
+     * @param gradientNorm The gradient norm to be used for the given iteration.
+     * @param offset The batch offset to be used for the given iteration.
+     * @param batchSize Batch size to be used for the given iteration.
+     * @param backtrackingBatchSize Backtracking batch size to be used for the
+     *        given iteration.
+     * @param reset Reset the step size decay parameter.
+     */
+    template<typename SeparableFunctionType,
+             typename GradType>
+    void Update(SeparableFunctionType& function,
+                double& stepSize,
+                MatType& iterate,
+                GradType& gradient,
+                double& gradientNorm,
+                double& /* sampleVariance */,
+                const size_t offset,
+                const size_t /* batchSize */,
+                const size_t backtrackingBatchSize,
+                const bool reset)
     {
-      stepSize /= 2;
+      if (reset)
+        stepSize *= 2;
 
-      iterateUpdate = iterate - (stepSize * gradient);
-      overallObjectiveUpdate = function.Evaluate(iterateUpdate,
-        offset, backtrackingBatchSize);
+      typedef typename MatType::elem_type ElemType;
+
+      ElemType overallObjective = function.Evaluate(iterate, offset,
+          backtrackingBatchSize);
+
+      MatType iterateUpdate = iterate - (stepSize * gradient);
+      ElemType overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
+          backtrackingBatchSize);
+
+      while (overallObjectiveUpdate >
+          (overallObjective - parent.searchParameter * stepSize *
+           gradientNorm))
+      {
+        stepSize /= 2;
+
+        iterateUpdate = iterate - (stepSize * gradient);
+        overallObjectiveUpdate = function.Evaluate(iterateUpdate,
+          offset, backtrackingBatchSize);
+      }
     }
-  }
+
+   private:
+    //! Reference to instantiated parent object.
+    BacktrackingLineSearch& parent;
+  };
 
  private:
   //! The search parameter for each iteration.

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp

@@ -58,7 +58,7 @@ namespace ens {
  *   title   = {Big Batch {SGD:} Automated Inference using Adaptive Batch
  *              Sizes},
  *   author  = {Soham De and Abhay Kumar Yadav and David W. Jacobs and
-                Tom Goldstein},
+ *              Tom Goldstein},
  *   journal = {CoRR},
  *   year    = {2017},
  *   url     = {http://arxiv.org/abs/1610.05792},
@@ -91,26 +91,58 @@ class BigBatchSGD
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param shuffle If true, the batch order is shuffled; otherwise, each
    *        batch is visited in linear order.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   BigBatchSGD(const size_t batchSize = 1000,
               const double stepSize = 0.01,
               const double batchDelta = 0.1,
               const size_t maxIterations = 100000,
               const double tolerance = 1e-5,
-              const bool shuffle = true);
+              const bool shuffle = true,
+              const bool exactObjective = false);
+
+  /**
+   * Clean any memory associated with the BigBatchSGD object.
+   */
+  ~BigBatchSGD();
+
   /**
    * Optimize the given function using big-batch SGD.  The given starting point
    * will be modified to store the finishing point of the algorithm, and the
    * final objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function,
-                  arma::mat& iterate);
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the batch size.
   size_t BatchSize() const { return batchSize; }
@@ -147,6 +179,11 @@ class BigBatchSGD
   //! Modify the update policy.
   UpdatePolicyType& UpdatePolicy() { return updatePolicy; }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return exactObjective; }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return exactObjective; }
+
  private:
   //! The size of the current batch.
   size_t batchSize;
@@ -167,8 +204,14 @@ class BigBatchSGD
   //! iterating.
   bool shuffle;
 
+  //! Controls whether or not the actual Objective value is calculated.
+  bool exactObjective;
+
   //! The update policy used to update the parameters in each iteration.
   UpdatePolicyType updatePolicy;
+
+  //! Instantiated update policy.
+  Any instUpdatePolicy;
 };
 
 using BBS_Armijo = BigBatchSGD<BacktrackingLineSearch>;

include/ensmallen_bits/bigbatch_sgd/bigbatch_sgd_impl.hpp

@@ -26,76 +26,88 @@ BigBatchSGD<UpdatePolicyType>::BigBatchSGD(
     const double batchDelta,
     const size_t maxIterations,
     const double tolerance,
-    const bool shuffle) :
+    const bool shuffle,
+    const bool exactObjective) :
     batchSize(batchSize),
     stepSize(stepSize),
     batchDelta(batchDelta),
     maxIterations(maxIterations),
     tolerance(tolerance),
     shuffle(shuffle),
+    exactObjective(exactObjective),
     updatePolicy(UpdatePolicyType())
 { /* Nothing to do. */ }
 
+template<typename UpdatePolicyType>
+BigBatchSGD<UpdatePolicyType>::~BigBatchSGD()
+{
+  instUpdatePolicy.Clean();
+}
+
 //! Optimize the function (minimize).
 template<typename UpdatePolicyType>
-template<typename DecomposableFunctionType>
-double BigBatchSGD<UpdatePolicyType>::Optimize(
-    DecomposableFunctionType& function, arma::mat& iterate)
+template<typename SeparableFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+BigBatchSGD<UpdatePolicyType>::Optimize(
+    SeparableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
 {
-  typedef Function<DecomposableFunctionType> FullFunctionType;
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<SeparableFunctionType, BaseMatType, BaseGradType>
+      FullFunctionType;
   FullFunctionType& f(static_cast<FullFunctionType&>(function));
 
   // Make sure we have all the methods that we need.
-  traits::CheckDecomposableFunctionTypeAPI<FullFunctionType>();
+  traits::CheckSeparableFunctionTypeAPI<FullFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  typedef typename UpdatePolicyType::template Policy<BaseMatType>
+      InstUpdatePolicyType;
+
+  if (!instUpdatePolicy.Has<InstUpdatePolicyType>())
+  {
+    instUpdatePolicy.Clean();
+    instUpdatePolicy.Set<InstUpdatePolicyType>(
+        new InstUpdatePolicyType(updatePolicy));
+  }
 
   // Find the number of functions to use.
   const size_t numFunctions = f.NumFunctions();
 
   // To keep track of where we are and how things are going.
   size_t currentFunction = 0;
-  double overallObjective = 0;
-  double lastObjective = DBL_MAX;
+  size_t epoch = 1;
+  ElemType overallObjective = 0;
+  ElemType lastObjective = DBL_MAX;
   bool reset = false;
-  arma::mat delta0, delta1;
+  BaseGradType delta0, delta1;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
   // Now iterate!
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  arma::mat functionGradient(iterate.n_rows, iterate.n_cols);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+  BaseGradType functionGradient(iterate.n_rows, iterate.n_cols);
   const size_t actualMaxIterations = (maxIterations == 0) ?
       std::numeric_limits<size_t>::max() : maxIterations;
-  for (size_t i = 0; i < actualMaxIterations; /* incrementing done manually */)
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t i = 0; i < actualMaxIterations && !terminate;
+      /* incrementing done manually */)
   {
-    // Is this iteration the start of a sequence?
-    if ((currentFunction % numFunctions) == 0 && i > 0)
-    {
-      // Output current objective function.
-      Info << "Big-batch SGD: iteration " << i << ", objective "
-          << overallObjective << "." << std::endl;
-
-      if (std::isnan(overallObjective) || std::isinf(overallObjective))
-      {
-        Warn << "Big-batch SGD: converged to " << overallObjective
-            << "; terminating with failure.  Try a smaller step size?"
-            << std::endl;
-        return overallObjective;
-      }
-
-      if (std::abs(lastObjective - overallObjective) < tolerance)
-      {
-        Info << "Big-batch SGD: minimized within tolerance " << tolerance
-            << "; terminating optimization." << std::endl;
-        return overallObjective;
-      }
-
-      // Reset the counter variables.
-      lastObjective = overallObjective;
-      overallObjective = 0;
-      currentFunction = 0;
-
-      if (shuffle) // Determine order of visitation.
-        f.Shuffle();
-    }
-
     // Find the effective batch size; we have to take the minimum of three
     // things:
     // - the batch size can't be larger than the user-specified batch size;
@@ -112,10 +124,16 @@ double BigBatchSGD<UpdatePolicyType>::Optimize(
     // Compute the stochastic gradient estimation.
     f.Gradient(iterate, currentFunction, gradient, 1);
 
+    terminate |= Callback::Gradient(*this, f, iterate, gradient, callbacks...);
+
     delta1 = gradient;
     for (size_t j = 1; j < effectiveBatchSize; ++j, ++k)
     {
       f.Gradient(iterate, currentFunction + j, functionGradient, 1);
+
+      terminate |= Callback::Gradient(*this, f, iterate, functionGradient,
+          callbacks...);
+
       delta0 = delta1 + (functionGradient - delta1) / k;
 
       // Compute sample variance.
@@ -150,6 +168,9 @@ double BigBatchSGD<UpdatePolicyType>::Optimize(
         for (size_t j = 0; j < batchOffset; ++j, ++k)
         {
           f.Gradient(iterate, batchStart + j, functionGradient, 1);
+          terminate |= Callback::Gradient(*this, f, iterate,
+              functionGradient, callbacks...);
+
           delta0 = delta1 + (functionGradient - delta1) / (k + 1);
 
           // Compute sample variance.
@@ -170,29 +191,83 @@ double BigBatchSGD<UpdatePolicyType>::Optimize(
       }
     }
 
-    updatePolicy.Update(f, stepSize, iterate, gradient, gB, vB,
-        currentFunction, batchSize, effectiveBatchSize, reset);
+    instUpdatePolicy.As<InstUpdatePolicyType>().Update(f, stepSize, iterate,
+        gradient, gB, vB, currentFunction, batchSize, effectiveBatchSize,
+        reset);
 
     // Update the iterate.
     iterate -= stepSize * gradient;
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
 
-    overallObjective += f.Evaluate(iterate, currentFunction,
+    const ElemType objective = f.Evaluate(iterate, currentFunction,
         effectiveBatchSize);
+    overallObjective += objective;
+
+    terminate |= Callback::Evaluate(*this, f, iterate, objective,
+        callbacks...);
 
     i += effectiveBatchSize;
     currentFunction += effectiveBatchSize;
+
+    // Is this iteration the start of a sequence?
+    if ((currentFunction % numFunctions) == 0)
+    {
+      terminate |= Callback::EndEpoch(*this, f, iterate, epoch++,
+          overallObjective / (ElemType) numFunctions, callbacks...);
+
+      // Output current objective function.
+      Info << "Big-batch SGD: iteration " << i << ", objective "
+          << overallObjective << "." << std::endl;
+
+      if (std::isnan(overallObjective) || std::isinf(overallObjective))
+      {
+        Warn << "Big-batch SGD: converged to " << overallObjective
+            << "; terminating with failure.  Try a smaller step size?"
+            << std::endl;
+
+        Callback::EndOptimization(*this, f, iterate, callbacks...);
+        return overallObjective;
+      }
+
+      if (std::abs(lastObjective - overallObjective) < tolerance ||
+          Callback::BeginEpoch(*this, f, iterate, epoch, overallObjective,
+              callbacks...))
+      {
+        Info << "Big-batch SGD: minimized within tolerance " << tolerance
+            << "; terminating optimization." << std::endl;
+
+        Callback::EndOptimization(*this, f, iterate, callbacks...);
+        return overallObjective;
+      }
+
+      // Reset the counter variables.
+      lastObjective = overallObjective;
+      overallObjective = 0;
+      currentFunction = 0;
+
+      if (shuffle) // Determine order of visitation.
+        f.Shuffle();
+    }
   }
 
   Info << "Big-batch SGD: maximum iterations (" << maxIterations << ") "
       << "reached; terminating optimization." << std::endl;
 
-  // Calculate final objective.
-  overallObjective = 0;
-  for (size_t i = 0; i < numFunctions; i += batchSize)
+  // Calculate final objective if exactObjective is set to true.
+  if (exactObjective)
   {
-    const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
-    overallObjective += f.Evaluate(iterate, i, effectiveBatchSize);
+    overallObjective = 0;
+    for (size_t i = 0; i < numFunctions; i += batchSize)
+    {
+      const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
+      const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+    }
   }
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/callbacks/callbacks.hpp

@@ -0,0 +1,776 @@
+/**
+ * @file callbacks.hpp
+ * @author Marcus Edel
+ *
+ * The Callback class will invoke the specified callbacks.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_CALLBACKS_HPP
+#define ENSMALLEN_CALLBACKS_CALLBACKS_HPP
+
+#include <ensmallen_bits/callbacks/traits.hpp>
+
+namespace ens {
+
+/**
+ * Callbacks are a set of functions that can be applied at given stages of the
+ * optimization process. The following callbacks are available:
+ *
+ * - Evaluate(optimizer, function, coordinates, objective):
+ *   called after any call to Evaluate().
+ *
+ * - StepTaken(optimizer, function, coordinates):
+ *   called after any step is taken.
+ *
+ * - Gradient(optimizer, function, coordinates, gradient):
+ *   called whenever the gradient is computed.
+ *
+ * - BeginEpoch(optimizer, function, coordinates, epoch, objective):
+ *   called at the beginning of a pass over the data. The objective may be
+ *   exact or an estimate depending on exactObjective's value.
+ *
+ * - EvaluateConstraint(optimizer, function, coordinates, constraint,
+ *                      constraintValue):
+ *   called after any call to EvaluateConstraint().
+ *
+ * - GradientConstraint(optimizer, function, coordinates, constraint,
+ *                      constraintGradient):
+ *   called after any call to GradientConstraint().
+ *
+ * - BeginOptimization(optimizer, function, coordinates):
+ *   called at the beginning of the optimization.
+ *
+ * - EndOptimization(optimizer, function, coordinates):
+ *   called at the end of the optimization.
+ */
+class Callback
+{
+ public:
+  /**
+   * Invoke the BeginOptimization() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasBeginOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
+      bool>::type
+  BeginOptimizationFunction(CallbackType& callback,
+                            OptimizerType& optimizer,
+                            FunctionType& function,
+                            MatType& coordinates)
+  {
+    return const_cast<CallbackType&>(callback).BeginOptimization(optimizer,
+        function, coordinates);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasBeginOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
+      bool>::type
+  BeginOptimizationFunction(CallbackType& callback,
+                            OptimizerType& optimizer,
+                            FunctionType& function,
+                            MatType& coordinates)
+  {
+    const_cast<CallbackType&>(callback).BeginOptimization(optimizer, function,
+        coordinates);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasBeginOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
+      bool>::type
+  BeginOptimizationFunction(CallbackType& /* callback */,
+                            OptimizerType& /* optimizer */,
+                            FunctionType& /* function */,
+                            MatType& /* coordinates */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the BeginOptimization() callback if
+   * it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool BeginOptimization(OptimizerType& optimizer,
+                                FunctionType& function,
+                                MatType& coordinates,
+                                CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::BeginOptimizationFunction(callbacks, optimizer,
+            function, coordinates)... };
+     return result;
+  }
+
+  /**
+   * Invoke the EndOptimization() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEndOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EndOptimizationFunction(CallbackType& callback,
+                          OptimizerType& optimizer,
+                          FunctionType& function,
+                          MatType& coordinates)
+  {
+    return (const_cast<CallbackType&>(callback).EndOptimization(
+        optimizer, function, coordinates), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<!callbacks::traits::HasEndOptimizationSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EndOptimizationFunction(CallbackType& /* callback */,
+                          OptimizerType& /* optimizer */,
+                          FunctionType& /* function */,
+                          MatType& /* coordinates */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the EndOptimization() callback if it
+   * exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool EndOptimization(OptimizerType& optimizer,
+                              FunctionType& function,
+                              MatType& coordinates,
+                              CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EndOptimizationFunction(callbacks, optimizer,
+            function, coordinates)... };
+     return result;
+  }
+
+  /**
+   * Invoke the Evaluate() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEvaluateSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EvaluateFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   const double objective)
+  {
+    return (const_cast<CallbackType&>(callback).Evaluate(
+        optimizer, function, coordinates, objective), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<!callbacks::traits::HasEvaluateSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EvaluateFunction(CallbackType& /* callback */,
+                   OptimizerType& /* optimizer */,
+                   FunctionType& /* function */,
+                   const MatType& /* coordinates */,
+                   const double /* objective */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the Evaluate() callback if it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool Evaluate(OptimizerType& optimizer,
+                       FunctionType& function,
+                       const MatType& coordinates,
+                       const double objective,
+                       CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)(objective);  // prevent spurious compiler warnings
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EvaluateFunction(callbacks, optimizer, function,
+        coordinates, objective)... };
+     return result;
+  }
+
+  /**
+   * Invoke the EvaluateConstraint() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param constraint The index of the constraint.
+   * @param constraintValue Constraint value of the current point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasEvaluateConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EvaluateConstraintFunction(CallbackType& callback,
+                             OptimizerType& optimizer,
+                             FunctionType& function,
+                             const MatType& coordinates,
+                             const size_t constraint,
+                             const double constraintValue)
+  {
+    return (const_cast<CallbackType&>(callback).EvaluateConstraint(
+        optimizer, function, coordinates, constraint, constraintValue), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      !callbacks::traits::HasEvaluateConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value,
+      bool>::type
+  EvaluateConstraintFunction(CallbackType& /* callback */,
+                             OptimizerType& /* optimizer */,
+                             FunctionType& /* function */,
+                             const MatType& /* coordinates */,
+                             const size_t /* constraint */,
+                             const double /* constraintValue */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the EvaluateConstraint() callback if
+   * it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param constraint The index of the constraint.
+   * @param constraintValue Constraint value of the current point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool EvaluateConstraint(OptimizerType& optimizer,
+                                 FunctionType& function,
+                                 const MatType& coordinates,
+                                 const size_t constraint,
+                                 const double constraintValue,
+                                 CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)(constraint);  // prevent spurious compiler warnings
+    (void)(constraintValue);
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EvaluateConstraintFunction(callbacks, optimizer,
+            function, coordinates, constraint, constraintValue)... };
+     return result;
+  }
+
+  /**
+   * Invoke the Gradient() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<callbacks::traits::HasGradientSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      bool>::type
+  GradientFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   GradType& gradient)
+  {
+    return (const_cast<CallbackType&>(callback).Gradient(
+        optimizer, function, coordinates, gradient), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<!callbacks::traits::HasGradientSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      bool>::type
+  GradientFunction(CallbackType& /* callback */,
+                   OptimizerType& /* optimizer */,
+                   FunctionType& /* function */,
+                   const MatType& /* coordinates */,
+                   GradType& /* gradient */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the Gradient() callback if it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  static bool Gradient(OptimizerType& optimizer,
+                       FunctionType& function,
+                       const MatType& coordinates,
+                       GradType& gradient,
+                       CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::GradientFunction(callbacks, optimizer, function,
+        coordinates, gradient)... };
+     return result;
+  }
+
+  /**
+   * Invoke the GradientConstraint() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<
+      callbacks::traits::HasGradientConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      bool>::type
+  GradientConstraintFunction(CallbackType& callback,
+                             OptimizerType& optimizer,
+                             FunctionType& function,
+                             const MatType& coordinates,
+                             const size_t constraint,
+                             GradType& gradient)
+  {
+    return (const_cast<CallbackType&>(callback).GradientConstraint(
+        optimizer, function, coordinates, constraint, gradient), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType>
+  static typename std::enable_if<
+      !callbacks::traits::HasGradientConstraintSignature<
+      CallbackType, OptimizerType, FunctionType, MatType, GradType>::value,
+      bool>::type
+  GradientConstraintFunction(CallbackType& /* callback */,
+                             OptimizerType& /* optimizer */,
+                             FunctionType& /* function */,
+                             const MatType& /* coordinates */,
+                             const size_t /* constraint */,
+                             GradType& /* gradient */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the GradientConstraint() callback if
+   * it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param gradient Matrix that holds the gradient.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  static bool Gradient(OptimizerType& optimizer,
+                       FunctionType& function,
+                       const MatType& coordinates,
+                       const size_t constraint,
+                       GradType& gradient,
+                       CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)(constraint);  // prevent spurious compiler warnings
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::GradientConstraintFunction(callbacks, optimizer,
+        function, coordinates, constraint, gradient)... };
+     return result;
+  }
+
+  /**
+   * Iterate over the callbacks and invoke the Evaluate() and Gradient()
+   * callback if it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   * @param gradient Matrix that holds the gradient.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  static bool EvaluateWithGradient(OptimizerType& optimizer,
+                                   FunctionType& function,
+                                   const MatType& coordinates,
+                                   const double objective,
+                                   GradType& gradient,
+                                   CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)(objective);  // prevent spurious compiler warnings
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EvaluateFunction(callbacks, optimizer, function,
+        coordinates, objective)... };
+
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::GradientFunction(callbacks, optimizer, function,
+        coordinates, gradient)... };
+     return result;
+  }
+
+  /**
+   * Invoke the BeginEpoch() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasBeginEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value, bool>::type
+  BeginEpochFunction(CallbackType& callback,
+                     OptimizerType& optimizer,
+                     FunctionType& function,
+                     const MatType& coordinates,
+                     const size_t epoch,
+                     const double objective)
+  {
+    return (const_cast<CallbackType&>(callback).BeginEpoch(
+        optimizer, function, coordinates, epoch, objective), false);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<!callbacks::traits::HasBeginEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::value, bool>::type
+  BeginEpochFunction(CallbackType& /* callback  */,
+                     OptimizerType& /* optimizer */,
+                     FunctionType& /* function  */,
+                     const MatType& /* coordinates  */,
+                     const size_t /* epoch  */,
+                     const double /* objective */)
+  { return false; }
+
+  /**
+   * Iterate over all callbacks and invoke the BeginEpoch() callback if it
+   * exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool BeginEpoch(OptimizerType& optimizer,
+                         FunctionType& function,
+                         const MatType& coordinates,
+                         const size_t epoch,
+                         const double objective,
+                         CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)(epoch);  // prevent spurious compiler warnings
+    (void)(objective);
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::BeginEpochFunction(callbacks, optimizer, function,
+        coordinates, epoch, objective)... };
+     return result;
+  }
+
+  /**
+   * Invoke the EndEpoch() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEndEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool, bool>::type
+  EndEpochFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   const size_t epoch,
+                   const double objective)
+  {
+    return const_cast<CallbackType&>(callback).EndEpoch(
+        optimizer, function, coordinates, epoch, objective);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEndEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid, bool>::type
+  EndEpochFunction(CallbackType& callback,
+                   OptimizerType& optimizer,
+                   FunctionType& function,
+                   const MatType& coordinates,
+                   const size_t epoch,
+                   const double objective)
+  {
+    const_cast<CallbackType&>(callback).EndEpoch(
+        optimizer, function, coordinates, epoch, objective);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<callbacks::traits::HasEndEpochSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone, bool>::type
+  EndEpochFunction(CallbackType& /* callback */,
+                   OptimizerType& /* optimizer */,
+                   FunctionType& /* function */,
+                   const MatType& /* coordinates */,
+                   const size_t /* epoch */,
+                   const double /* objective */)
+  { return false; }
+
+  /**
+   * Iterate over all callbacks and invoke the EndEpoch() callback if it exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool EndEpoch(OptimizerType& optimizer,
+                       FunctionType& function,
+                       const MatType& coordinates,
+                       const size_t epoch,
+                       const double objective,
+                       CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)(epoch);  // prevent spurious compiler warnings
+    (void)(objective);
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::EndEpochFunction(callbacks, optimizer, function,
+        coordinates, epoch, objective)... };
+     return result;
+  }
+
+  /**
+   * Invoke the StepTaken() callback if it exists.
+   *
+   * @param callback The callback to call.
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasBool,
+      bool>::type
+  StepTakenFunction(CallbackType& callback,
+                    OptimizerType& optimizer,
+                    FunctionType& function,
+                    MatType& coordinates)
+  {
+    return const_cast<CallbackType&>(callback).StepTaken(optimizer,
+        function, coordinates);
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasVoid,
+      bool>::type
+  StepTakenFunction(CallbackType& callback,
+                    OptimizerType& optimizer,
+                    FunctionType& function,
+                    MatType& coordinates)
+  {
+    const_cast<CallbackType&>(callback).StepTaken(optimizer, function,
+        coordinates);
+    return false;
+  }
+
+  template<typename CallbackType,
+           typename OptimizerType,
+           typename FunctionType,
+           typename MatType>
+  static typename std::enable_if<
+      callbacks::traits::HasStepTakenSignature<
+      CallbackType, OptimizerType, FunctionType, MatType>::hasNone,
+      bool>::type
+  StepTakenFunction(CallbackType& /* callback */,
+                    OptimizerType& /* optimizer */,
+                    FunctionType& /* function */,
+                    MatType& /* coordinates */)
+  { return false; }
+
+  /**
+   * Iterate over the callbacks and invoke the StepTaken() callback if it
+   * exists.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param callbacks The callbacks container.
+   */
+  template<typename OptimizerType,
+           typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  static bool StepTaken(OptimizerType& optimizer,
+                        FunctionType& function,
+                        MatType& coordinates,
+                        CallbackTypes&... callbacks)
+  {
+    // This will return immediately once a callback returns true.
+    bool result = false;
+    (void)std::initializer_list<bool>{ result =
+        result || Callback::StepTakenFunction(callbacks, optimizer,
+            function, coordinates)... };
+     return result;
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/early_stop_at_min_loss.hpp

@@ -0,0 +1,127 @@
+/**
+ * @file early_stop_at_min_loss.hpp
+ * @author Marcus Edel
+ * @author Omar Shrit
+ *
+ * Implementation of the early stop at minimum loss callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_EARLY_STOP_AT_MIN_LOSS_HPP
+#define ENSMALLEN_CALLBACKS_EARLY_STOP_AT_MIN_LOSS_HPP
+
+#include <functional>
+
+namespace ens {
+
+/**
+ * Early stopping to terminate the optimization process early if the loss stops
+ * decreasing.
+ */
+template<typename MatType = arma::mat>
+class EarlyStopAtMinLossType
+{
+ public:
+  /**
+   * Set up the early stop at min loss class, which keeps track of the minimum
+   * loss and stops the optimization process if the loss stops decreasing.
+   *
+   * @param patienceIn The number of epochs to wait after the minimum loss has
+   *    been reached or no improvement has been made (Default: 10).
+   */
+  EarlyStopAtMinLossType<MatType>(const size_t patienceIn = 10) :
+      callbackUsed(false), 
+      patience(patienceIn), 
+      bestObjective(std::numeric_limits<double>::max()),
+      steps(0)
+  { /* Nothing to do here */ }
+
+  /**
+   * Set up the early stop at min loss class, which keeps track of the minimum
+   * loss and stops the optimization process if the loss stops decreasing.
+   *
+   * @param func, callback to return immediate loss evaluated by the function
+   * @param patienceIn The number of epochs to wait after the minimum loss has
+   *    been reached or no improvement has been made (Default: 10).
+   */
+  EarlyStopAtMinLossType<MatType>(
+      std::function<double(const MatType&)> func,
+      const size_t patienceIn = 10)
+    : callbackUsed(true), 
+      patience(patienceIn), 
+      bestObjective(std::numeric_limits<double>::max()),
+      steps(0), 
+      localFunc(func)
+  {
+    // Nothing to do here
+  }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType>
+  bool EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& coordinates,
+                const size_t /* epoch */,
+                double objective)
+  {
+    if (callbackUsed)
+    {
+      objective = localFunc(coordinates);
+    } 
+
+    if (objective < bestObjective)
+    {
+      steps = 0;
+      bestObjective = objective;
+      return false;
+    }
+
+    steps++;
+    if (steps >= patience)
+    {
+      Info << "Minimum loss reached; terminate optimization." << std::endl;
+      return true;
+    }
+
+    return false;
+  }
+
+ private:
+  //! False if the first constructor is called, true if the user passed a lambda. 
+  bool callbackUsed;
+
+  //! The number of epochs to wait before terminating the optimization process.
+  size_t patience;
+
+  //! Locally-stored best objective.
+  double bestObjective;
+
+  //! Locally-stored number of steps since the loss improved.
+  size_t steps;
+
+  //! Function to call at the end of the epoch.
+  std::function<double(const MatType&)> localFunc;
+};
+
+/*
+ * Note that the using definition is temporary, this definition should
+ * be removed when releasing ensmallen 3.0
+ * The renaming of the class is only to avoid a major version bump
+ * because if the template type added to this class
+ */
+using EarlyStopAtMinLoss = EarlyStopAtMinLossType<arma::mat>;
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/print_loss.hpp

@@ -0,0 +1,57 @@
+/**
+ * @file print_loss.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the print loss callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_PRINT_LOSS_HPP
+#define ENSMALLEN_CALLBACKS_PRINT_LOSS_HPP
+
+namespace ens {
+
+/**
+ * Print loss function, based on the EndEpoch callback function.
+ */
+class PrintLoss
+{
+ public:
+  /**
+   * Set up the print loss callback class with the width and output stream.
+   *
+   * @param ostream Ostream which receives output from this object.
+   */
+  PrintLoss(std::ostream& output = arma::get_cout_stream()) : output(output)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double objective)
+  {
+    output << objective << std::endl;
+  }
+
+ private:
+  //! The output stream that all data is to be sent to; example: std::cout.
+  std::ostream& output;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/progress_bar.hpp

@@ -0,0 +1,263 @@
+/**
+ * @file progress_bar.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of a simple progress bar callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_PROGRESS_BAR_HPP
+#define ENSMALLEN_CALLBACKS_PROGRESS_BAR_HPP
+
+#include <ensmallen_bits/function.hpp>
+
+namespace ens {
+
+/**
+ * A simple progress bar, based on the maximum number of optimizer iterations,
+ * batch-size, number of functions and the StepTaken callback function.
+ */
+class ProgressBar
+{
+ public:
+  /**
+   * Set up the progress bar callback class with the given width and output
+   * stream.
+   *
+   * @param widthIn Width of the bar.
+   * @param ostream Ostream which receives output from this object.
+   */
+  ProgressBar(const size_t widthIn = 70,
+              std::ostream& output = arma::get_cout_stream()) :
+      width(100.0 / widthIn),
+      output(output),
+      objective(0),
+      epochs(0),
+      epochSize(0),
+      step(1),
+      steps(0),
+      newEpoch(false),
+      epoch(1)
+
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at the begin of the optimization process.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginOptimization(OptimizerType& optimizer,
+                         FunctionType& function,
+                         MatType& /* coordinates */)
+  {
+    static_assert(traits::HasBatchSizeSignature<
+      OptimizerType>::value,
+      "The OptimizerType does not have a correct definition of BatchSize(). "
+      "Please check that the OptimizerType fully satisfies the requirements "
+      "of the ProgressBar API; see the callbacks documentation for more "
+      "details.");
+
+    static_assert(traits::HasMaxIterationsSignature<
+      OptimizerType>::value,
+      "The OptimizerType does not have a correct definition of MaxIterations()."
+      " Please check that the OptimizerType fully satisfies the requirements "
+      "of the ProgressBar API; see the callbacks documentation for more "
+      "details.");
+
+    static_assert(traits::HasNumFunctionsSignature<
+      FunctionType>::value,
+      "The OptimizerType does not have a correct definition of NumFunctions(). "
+      "Please check that the OptimizerType fully satisfies the requirements "
+      "of the ProgressBar API; see the callbacks documentation for more "
+      "details.");
+
+    epochSize = function.NumFunctions() / optimizer.BatchSize();
+    if (function.NumFunctions() % optimizer.BatchSize() > 0)
+      epochSize++;
+
+    epochs = optimizer.MaxIterations() / function.NumFunctions();
+    if (optimizer.MaxIterations() % function.NumFunctions() > 0)
+      epochs++;
+
+    stepTimer.tic();
+  }
+
+  /**
+   * Callback function called at the begin of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epochIn The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginEpoch(OptimizerType& /* optimizer */,
+                  FunctionType& /* function */,
+                  const MatType& /* coordinates */,
+                  const size_t epochIn,
+                  const double /* objective */)
+  {
+    // Start the timer.
+    epochTimer.tic();
+
+    // Reset epoch parameter.
+    objective = 0;
+    step = 1;
+
+    epoch = epochIn;
+    newEpoch = true;
+  }
+
+  /**
+   * Callback function called once a step is taken.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void StepTaken(OptimizerType& /* optimizer */,
+                 FunctionType& /* function */,
+                 const MatType& /* coordinates */)
+  {
+    if (newEpoch)
+    {
+      output << "Epoch " << epoch;
+      if (epochs > 0)
+      {
+        output << "/" << epochs;
+      }
+      output << '\n';
+      newEpoch = false;
+    }
+
+    const size_t progress = ((double) step / epochSize) * 100;
+    output << step++ << "/" << epochSize << " [";
+    for (size_t i = 0; i < 100; i += width)
+    {
+      if (i < progress)
+      {
+        output << "=";
+      }
+      else if (i == progress)
+      {
+        output << ">";
+      }
+      else
+      {
+        output << ".";
+      }
+    }
+
+    output << "] " << progress << "% - ETA: " << (size_t) (stepTimer.toc() *
+        (epochSize - step + 1)) % 60 << "s - loss: " <<
+        objective / (double) step <<  "\r";
+    output.flush();
+
+    stepTimer.tic();
+  }
+
+  /**
+   * Callback function called at any call to Evaluate().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objectiveIn Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Evaluate(OptimizerType& optimizer,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const double objectiveIn)
+  {
+    objective += objectiveIn / optimizer.BatchSize();
+    steps++;
+  }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double objective)
+  {
+    const size_t progress = ((double) (step - 1) / epochSize) * 100;
+    output << step - 1 << "/" << epochSize << " [";
+    for (size_t i = 0; i < 100; i += width)
+    {
+      if (i < progress)
+      {
+        output << "=";
+      }
+      else if (i == progress)
+      {
+        output << ">";
+      }
+      else
+      {
+        output << ".";
+      }
+    }
+
+    const size_t stepTime = epochTimer.toc() / (double) epochSize * 1000;
+    output << "] " << progress << "% - " << (size_t) epochTimer.toc() % 60
+        << "s " << stepTime << "ms/step " << "- loss: " << objective  <<  "\n";
+    output.flush();
+  }
+
+ private:
+  //! Length of a single step (1%).
+  double width;
+
+  //! The output stream that all data is to be sent to; example: std::cout.
+  std::ostream& output;
+
+  //! Objective over the current epoch.
+  double objective;
+
+  //! Total number of epochs
+  size_t epochs;
+
+  //! Number of steps per epoch.
+  size_t epochSize;
+
+  //! Current step number.
+  size_t step;
+
+  //! Number of steps taken.
+  size_t steps;
+
+  //! Indicates a new epoch.
+  bool newEpoch;
+
+  //! Locally-stored epoch.
+  size_t epoch;
+
+  //! Locally-stored step timer object.
+  arma::wall_clock stepTimer;
+
+  //! Locally-stored epoch timer object.
+  arma::wall_clock epochTimer;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/store_best_coordinates.hpp

@@ -0,0 +1,75 @@
+/**
+ * @file store_best_coordinates.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the store best coordinates callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_STORE_BEST_COORDINATES_HPP
+#define ENSMALLEN_CALLBACKS_STORE_BEST_COORDINATES_HPP
+
+namespace ens {
+
+/**
+ * Store best coordinates function, based on the Evaluate callback function.
+ *
+ * @tparam MatType Type of the model coordinates (arma::colvec, arma::mat,
+ *     arma::sp_mat or arma::cube).
+ */
+template<typename ModelMatType = arma::mat>
+class StoreBestCoordinates
+{
+ public:
+  /**
+   * Set up the store best model class, which keeps the best-performing
+   * coordinates and objective.
+   */
+  StoreBestCoordinates() : bestObjective(std::numeric_limits<double>::max())
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called after any call to Evaluate().
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void Evaluate(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& coordinates,
+                const double objective)
+  {
+    if (objective < bestObjective)
+    {
+      bestObjective = objective;
+      bestCoordinates = coordinates;
+    }
+  }
+
+  //! Get the best coordinates.
+  ModelMatType const& BestCoordinates() const { return bestCoordinates; }
+  //! Modify the best coordinates.
+  ModelMatType& BestCoordinatesl() { return bestCoordinates; }
+
+  //! Get the best objective.
+  double const& BestObjective() const { return bestObjective; }
+  //! Modify the best objective.
+  double& BestObjective() { return bestObjective; }
+
+ private:
+  //! Locally-stored best objective.
+  double bestObjective;
+
+  //! Locally-stored best model coordinates.
+  ModelMatType bestCoordinates;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/timer_stop.hpp

@@ -0,0 +1,83 @@
+/**
+ * @file timer_stop.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the timer stop callback function.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_TIMER_STOP_HPP
+#define ENSMALLEN_CALLBACKS_TIMER_STOP_HPP
+
+namespace ens {
+
+/**
+ * Timer stop function, is based on the BeginOptimization callback function to
+ * start the timer and the EndEpoch callback function to update the timer.
+ */
+class TimerStop
+{
+ public:
+  /**
+   * Set up the print loss callback class with the width and output stream.
+   *
+   * @param durationIn The duration of the timer in seconds.
+   */
+  TimerStop(const double durationIn) : duration(durationIn)
+  { /* Nothing to do here. */ }
+
+  /**
+   * Callback function called at the start of the optimization process.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  void BeginOptimization(OptimizerType& /* optimizer */,
+                         FunctionType& /* function */,
+                         MatType& /* coordinates */)
+  {
+    // Start the timer.
+    timer.tic();
+  }
+
+  /**
+   * Callback function called at the end of a pass over the data.
+   *
+   * @param optimizer The optimizer used to update the function.
+   * @param function Function to optimize.
+   * @param coordinates Starting point.
+   * @param epoch The index of the current epoch.
+   * @param objective Objective value of the current point.
+   */
+  template<typename OptimizerType, typename FunctionType, typename MatType>
+  bool EndEpoch(OptimizerType& /* optimizer */,
+                FunctionType& /* function */,
+                const MatType& /* coordinates */,
+                const size_t /* epoch */,
+                const double /* objective */)
+  {
+    if (timer.toc() > duration)
+    {
+      Info << "Timer timeout reached; terminate optimization." << std::endl;
+      return true;
+    }
+
+    return false;
+  }
+
+ private:
+  //! The duration in seconds.
+  double duration;
+
+  //! Locally-stored timer object.
+  arma::wall_clock timer;
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/callbacks/traits.hpp

@@ -0,0 +1,372 @@
+/**
+ * @file traits.hpp
+ * @author Marcus Edel
+ *
+ * This file provides metaprogramming utilities for detecting certain members of
+ * CallbackType classes.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_CALLBACKS_TRAITS_HPP
+#define ENSMALLEN_CALLBACKS_TRAITS_HPP
+
+#include <ensmallen_bits/function/sfinae_utility.hpp>
+
+namespace ens {
+namespace callbacks {
+namespace traits {
+
+//! Detect an Evaluate() method.
+ENS_HAS_EXACT_METHOD_FORM(Evaluate, HasEvaluate)
+//! Detect an EvaluateConstraint() method.
+ENS_HAS_EXACT_METHOD_FORM(EvaluateConstraint, HasEvaluateConstraint)
+//! Detect an Gradient() method.
+ENS_HAS_EXACT_METHOD_FORM(Gradient, HasGradient)
+//! Detect an GradientConstraint() method.
+ENS_HAS_EXACT_METHOD_FORM(GradientConstraint, HasGradientConstraint)
+//! Detect an BeginOptimization() method.
+ENS_HAS_EXACT_METHOD_FORM(BeginOptimization, HasBeginOptimization)
+//! Detect an EndOptimization() method.
+ENS_HAS_EXACT_METHOD_FORM(EndOptimization, HasEndOptimization)
+//! Detect an BeginEpoch() method.
+ENS_HAS_EXACT_METHOD_FORM(BeginEpoch, HasBeginEpoch)
+//! Detect an EndEpoch() method.
+ENS_HAS_EXACT_METHOD_FORM(EndEpoch, HasEndEpoch)
+//! Detect an StepTaken() method.
+ENS_HAS_EXACT_METHOD_FORM(StepTaken, HasStepTaken)
+
+template<typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename GradType = MatType>
+struct TypedForms
+{
+  //! This is the form of a bool Evaluate() callback method.
+  template<typename CallbackType>
+  using EvaluateBoolForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const double);
+
+  //! This is the form of a void Evaluate() callback method.
+  template<typename CallbackType>
+  using EvaluateVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const double);
+
+  //! This is the form of a bool EvaluateConstraint() callback method.
+  template<typename CallbackType>
+  using EvaluateConstraintBoolForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a void EvaluateConstraint() callback method.
+  template<typename CallbackType>
+  using EvaluateConstraintVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a bool Gradient() callback method.
+  template<typename CallbackType>
+  using GradientBoolForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const MatType&);
+
+  //! This is the form of a void Gradient() callback method.
+  template<typename CallbackType>
+  using GradientVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const MatType&);
+
+  //! This is the form of a bool GradientConstraint() callback method.
+  template<typename CallbackType>
+  using GradientConstraintBoolForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const MatType&);
+
+  //! This is the form of a void GradientConstraint() callback method.
+  template<typename CallbackType>
+  using GradientConstraintVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const MatType&);
+
+  //! This is the form of a bool BeginOptimization() callback method.
+  template<typename CallbackType>
+  using BeginOptimizationBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            MatType&);
+
+  //! This is the form of a void BeginOptimization() callback method.
+  template<typename CallbackType>
+  using BeginOptimizationVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            MatType&);
+
+  //! This is the form of a bool EndOptimization() callback method.
+  template<typename CallbackType>
+  using EndOptimizationBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            MatType&);
+
+  //! This is the form of a void EndOptimization() callback method.
+  template<typename CallbackType>
+  using EndOptimizationVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            MatType&);
+
+  //! This is the form of a bool BeginEpoch() callback method.
+  template<typename CallbackType>
+  using BeginEpochBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a void BeginEpoch() callback method.
+  template<typename CallbackType>
+  using BeginEpochVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a bool EndEpoch() callback method.
+  template<typename CallbackType>
+  using EndEpochBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a void EndEpoch() callback method.
+  template<typename CallbackType>
+  using EndEpochVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&,
+                            const size_t,
+                            const double);
+
+  //! This is the form of a bool StepTaken() callback method.
+  template<typename CallbackType>
+  using StepTakenBoolForm =
+      bool(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&);
+
+  //! This is the form of a void StepTaken() callback method.
+  template<typename CallbackType>
+  using StepTakenVoidForm =
+      void(CallbackType::*)(OptimizerType&,
+                            FunctionType&,
+                            const MatType&);
+};
+
+//! Utility struct, check if either void BeginOptimization() or
+//! bool BeginOptimization() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasBeginOptimizationSignature
+{
+  const static bool hasBool =
+      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
+      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
+
+  const static bool hasVoid =
+      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
+      HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+         FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
+
+  const static bool hasNone =
+      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template BeginOptimizationBoolForm>::value &&
+      !HasBeginOptimization<CallbackType, TypedForms<OptimizerType,
+         FunctionType, MatType>::template BeginOptimizationVoidForm>::value;
+};
+
+//! Utility struct, check if either void Evaluate() or bool Evaluate()
+//! exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasEvaluateSignature
+{
+  const static bool value =
+      HasEvaluate<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EvaluateBoolForm>::value ||
+      HasEvaluate<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EvaluateVoidForm>::value;
+};
+
+//! Utility struct, check if either void EvaluateConstraint() or
+//! bool EvaluateConstraint() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasEvaluateConstraintSignature
+{
+  const static bool value =
+      HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EvaluateConstraintBoolForm>::value ||
+      HasEvaluateConstraint<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EvaluateConstraintVoidForm>::value;
+};
+
+//! Utility struct, check if either void Gradient() or bool Gradient()
+//! exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename Gradient>
+struct HasGradientSignature
+{
+  const static bool value =
+      HasGradient<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType, Gradient>::template GradientBoolForm>::value ||
+      HasGradient<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType, Gradient>::template GradientVoidForm>::value;
+};
+
+//! Utility struct, check if either void GradientConstraint() or
+//! bool GradientConstraint() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType,
+         typename Gradient>
+struct HasGradientConstraintSignature
+{
+  const static bool value =
+      HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType,
+          Gradient>::template GradientConstraintBoolForm>::value ||
+      HasGradientConstraint<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType,
+          Gradient>::template GradientConstraintVoidForm>::value;
+};
+
+//! Utility struct, check if either void EndOptimization() or
+//! bool EndOptimization() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasEndOptimizationSignature
+{
+  const static bool value =
+      HasEndOptimization<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EndOptimizationBoolForm>::value ||
+      HasEndOptimization<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template EndOptimizationVoidForm>::value;
+};
+
+//! Utility struct, check if either void BeginEpoch() or bool BeginEpoch()
+//! exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasBeginEpochSignature
+{
+  const static bool value =
+      HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template BeginEpochBoolForm>::value ||
+      HasBeginEpoch<CallbackType, TypedForms<OptimizerType,
+      FunctionType, MatType>::template BeginEpochVoidForm>::value;
+};
+
+//! Utility struct, check if either void EndEpoch() or bool EndEpoch()
+//! exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasEndEpochSignature
+{
+  const static bool hasBool =
+      HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochBoolForm>::value &&
+      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochVoidForm>::value;
+
+  const static bool hasVoid =
+      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochBoolForm>::value &&
+      HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochVoidForm>::value;
+
+  const static bool hasNone =
+      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochBoolForm>::value &&
+      !HasEndEpoch<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template EndEpochVoidForm>::value;
+};
+
+//! Utility struct, check if either void StepTaken() or bool StepTaken() exists.
+template<typename CallbackType,
+         typename OptimizerType,
+         typename FunctionType,
+         typename MatType>
+struct HasStepTakenSignature
+{
+  const static bool hasBool =
+      HasStepTaken<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template StepTakenBoolForm>::value &&
+      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template StepTakenVoidForm>::value;
+
+  const static bool hasVoid =
+      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template StepTakenBoolForm>::value &&
+      HasStepTaken<CallbackType, TypedForms<OptimizerType,
+         FunctionType, MatType>::template StepTakenVoidForm>::value;
+
+  const static bool hasNone =
+      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
+          FunctionType, MatType>::template StepTakenBoolForm>::value &&
+      !HasStepTaken<CallbackType, TypedForms<OptimizerType,
+         FunctionType, MatType>::template StepTakenVoidForm>::value;
+};
+
+} // namespace traits
+} // namespace callbacks
+} // namespace ens
+
+#endif

include/ensmallen_bits/cmaes/cmaes.hpp

@@ -82,13 +82,20 @@ class CMAES
    * modified to store the finishing point of the algorithm, and the final
    * objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
 
   //! Get the step size.
   size_t PopulationSize() const { return lambda; }

include/ensmallen_bits/cmaes/cmaes_impl.hpp

@@ -41,13 +41,24 @@ CMAES<SelectionPolicyType>::CMAES(const size_t lambda,
 
 //! Optimize the function (minimize).
 template<typename SelectionPolicyType>
-template<typename DecomposableFunctionType>
-double CMAES<SelectionPolicyType>::Optimize(
-    DecomposableFunctionType& function, arma::mat& iterate)
+template<typename SeparableFunctionType,
+         typename MatType,
+         typename... CallbackTypes>
+typename MatType::elem_type CMAES<SelectionPolicyType>::Optimize(
+    SeparableFunctionType& function,
+    MatType& iterateIn,
+    CallbackTypes&&... callbacks)
 {
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
   // Make sure that we have the methods that we need.  Long name...
-  traits::CheckNonDifferentiableDecomposableFunctionTypeAPI<
-      DecomposableFunctionType>();
+  traits::CheckArbitrarySeparableFunctionTypeAPI<
+      SeparableFunctionType, BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // Find the number of functions to use.
   const size_t numFunctions = function.NumFunctions();
@@ -58,15 +69,15 @@ double CMAES<SelectionPolicyType>::Optimize(
 
   // Parent weights.
   const size_t mu = std::round(lambda / 2);
-  arma::vec w = std::log(mu + 0.5) - arma::log(
-    arma::linspace<arma::vec>(0, mu - 1, mu) + 1.0);
-  w /= arma::sum(w);
+  BaseMatType w = std::log(mu + 0.5) - arma::log(
+      arma::linspace<BaseMatType>(0, mu - 1, mu) + 1.0);
+  w /= arma::accu(w);
 
   // Number of effective solutions.
   const double muEffective = 1 / arma::accu(arma::pow(w, 2));
 
   // Step size control parameters.
-  arma::vec sigma(3);
+  BaseMatType sigma(2, 1); // sigma is vector-shaped.
   sigma(0) = 0.3 * (upperBound - lowerBound);
   const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5);
   const double ds = 1 + cs + 2 * std::max(std::sqrt((muEffective - 1) /
@@ -86,139 +97,157 @@ double CMAES<SelectionPolicyType>::Optimize(
       muEffective) / (std::pow(iterate.n_elem + 2, 2) +
       alphaMu * muEffective / 2));
 
-  arma::cube mPosition(iterate.n_rows, iterate.n_cols, 3);
-  mPosition.slice(0) = lowerBound + arma::randu(
+  std::vector<BaseMatType> mPosition(2, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  mPosition[0] = lowerBound + arma::randu<BaseMatType>(
       iterate.n_rows, iterate.n_cols) * (upperBound - lowerBound);
 
-  arma::mat step = arma::zeros(iterate.n_rows, iterate.n_cols);
+  BaseMatType step(iterate.n_rows, iterate.n_cols);
+  step.zeros();
 
   // Calculate the first objective function.
-  double currentObjective = 0;
+  ElemType currentObjective = 0;
   for (size_t f = 0; f < numFunctions; f += batchSize)
   {
     const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
-    currentObjective += function.Evaluate(mPosition.slice(0), f,
+    const ElemType objective = function.Evaluate(mPosition[0], f,
         effectiveBatchSize);
+    currentObjective += objective;
+
+    Callback::Evaluate(*this, function, mPosition[0], objective,
+        callbacks...);
   }
 
-  double overallObjective = currentObjective;
-  double lastObjective = DBL_MAX;
+  ElemType overallObjective = currentObjective;
+  ElemType lastObjective = std::numeric_limits<ElemType>::max();
 
   // Population parameters.
-  arma::cube pStep(iterate.n_rows, iterate.n_cols, lambda);
-  arma::cube pPosition(iterate.n_rows, iterate.n_cols, lambda);
-  arma::vec pObjective(lambda);
-  arma::cube ps = arma::zeros(iterate.n_rows, iterate.n_cols, 2);
-  arma::cube pc = ps;
-  arma::cube C(iterate.n_elem, iterate.n_elem, 2);
-  C.slice(0).eye();
+  std::vector<BaseMatType> pStep(lambda, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  std::vector<BaseMatType> pPosition(lambda, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  BaseMatType pObjective(lambda, 1); // pObjective is vector-shaped.
+  std::vector<BaseMatType> ps(2, BaseMatType(iterate.n_rows, iterate.n_cols));
+  ps[0].zeros();
+  ps[1].zeros();
+  std::vector<BaseMatType> pc = ps;
+  std::vector<BaseMatType> C(2, BaseMatType(iterate.n_elem, iterate.n_elem));
+  C[0].eye();
 
   // Covariance matrix parameters.
-  arma::vec eigval;
-  arma::mat eigvec;
-  arma::vec eigvalZero = arma::zeros(iterate.n_elem);
+  arma::Col<ElemType> eigval; // TODO: might need a more general type.
+  BaseMatType eigvec;
+  BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped.
+  eigvalZero.zeros();
 
   // The current visitation order (sorted by population objectives).
   arma::uvec idx = arma::linspace<arma::uvec>(0, lambda - 1, lambda);
 
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
   // Now iterate!
-  for (size_t i = 1; i < maxIterations; ++i)
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t i = 1; i < maxIterations && !terminate; ++i)
   {
     // To keep track of where we are.
     const size_t idx0 = (i - 1) % 2;
     const size_t idx1 = i % 2;
 
-    const arma::mat covLower = arma::chol(C.slice(idx0), "lower");
+    // Perform Cholesky decomposition. If the matrix is not positive definite,
+    // add a small value and try again.
+    BaseMatType covLower;
+    while (!arma::chol(covLower, C[idx0], "lower"))
+      C[idx0].diag() += 1e-16;
 
     for (size_t j = 0; j < lambda; ++j)
     {
       if (iterate.n_rows > iterate.n_cols)
       {
-        pStep.slice(idx(j)) = covLower *
-            arma::randn(iterate.n_rows, iterate.n_cols);
+        pStep[idx(j)] = covLower *
+            arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols);
       }
       else
       {
-        pStep.slice(idx(j)) = arma::randn(iterate.n_rows, iterate.n_cols) *
-            covLower;
+        pStep[idx(j)] = arma::randn<BaseMatType>(iterate.n_rows, iterate.n_cols)
+            * covLower;
       }
 
-      pPosition.slice(idx(j)) = mPosition.slice(idx0) + sigma(idx0) *
-          pStep.slice(idx(j));
+      pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)];
 
       // Calculate the objective function.
       pObjective(idx(j)) = selectionPolicy.Select(function, batchSize,
-          pPosition.slice(idx(j)));
+          pPosition[idx(j)], callbacks...);
     }
 
     // Sort population.
-    idx = sort_index(pObjective);
+    idx = arma::sort_index(pObjective);
 
-    step = w(0) * pStep.slice(idx(0));
+    step = w(0) * pStep[idx(0)];
     for (size_t j = 1; j < mu; ++j)
-      step += w(j) * pStep.slice(idx(j));
+      step += w(j) * pStep[idx(j)];
 
-    mPosition.slice(idx1) = mPosition.slice(idx0) + sigma(idx0) * step;
+    mPosition[idx1] = mPosition[idx0] + sigma(idx0) * step;
 
     // Calculate the objective function.
     currentObjective = selectionPolicy.Select(function, batchSize,
-          mPosition.slice(idx1));
+        mPosition[idx1], callbacks...);
 
     // Update best parameters.
     if (currentObjective < overallObjective)
     {
       overallObjective = currentObjective;
-      iterate = mPosition.slice(idx1);
+      iterate = mPosition[idx1];
+
+      terminate |= Callback::StepTaken(*this, function, iterate, callbacks...);
     }
 
     // Update Step Size.
     if (iterate.n_rows > iterate.n_cols)
     {
-      ps.slice(idx1) = (1 - cs) * ps.slice(idx0) + std::sqrt(
+      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
           cs * (2 - cs) * muEffective) * covLower.t() * step;
     }
     else
     {
-      ps.slice(idx1) = (1 - cs) * ps.slice(idx0) + std::sqrt(
+      ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt(
           cs * (2 - cs) * muEffective) * step * covLower.t();
     }
 
-    const double psNorm = arma::norm(ps.slice(idx1));
-    sigma(idx1) = sigma(idx0) * std::pow(
-        std::exp(cs / ds * psNorm / enn - 1), 0.3);
+    const ElemType psNorm = arma::norm(ps[idx1]);
+    sigma(idx1) = sigma(idx0) * std::exp(cs / ds * ( psNorm / enn - 1));
 
     // Update covariance matrix.
     if ((psNorm / sqrt(1 - std::pow(1 - cs, 2 * i))) < h)
     {
-      pc.slice(idx1) = (1 - cc) * pc.slice(idx0) + std::sqrt(cc * (2 - cc) *
+      pc[idx1] = (1 - cc) * pc[idx0] + std::sqrt(cc * (2 - cc) *
         muEffective) * step;
 
-
       if (iterate.n_rows > iterate.n_cols)
       {
-        C.slice(idx1) = (1 - c1 - cmu) * C.slice(idx0) + c1 *
-          (pc.slice(idx1) * pc.slice(idx1).t());
+        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
+          (pc[idx1] * pc[idx1].t());
       }
       else
       {
-        C.slice(idx1) = (1 - c1 - cmu) * C.slice(idx0) + c1 *
-          (pc.slice(idx1).t() * pc.slice(idx1));
+        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
+          (pc[idx1].t() * pc[idx1]);
       }
     }
     else
     {
-      pc.slice(idx1) = (1 - cc) * pc.slice(idx0);
+      pc[idx1] = (1 - cc) * pc[idx0];
 
       if (iterate.n_rows > iterate.n_cols)
       {
-        C.slice(idx1) = (1 - c1 - cmu) * C.slice(idx0) + c1 * (pc.slice(idx1) *
-            pc.slice(idx1).t() + (cc * (2 - cc)) * C.slice(idx0));
+        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 * (pc[idx1] *
+            pc[idx1].t() + (cc * (2 - cc)) * C[idx0]);
       }
       else
       {
-        C.slice(idx1) = (1 - c1 - cmu) * C.slice(idx0) + c1 *
-            (pc.slice(idx1).t() * pc.slice(idx1) + (cc * (2 - cc)) *
-            C.slice(idx0));
+        C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 *
+            (pc[idx1].t() * pc[idx1] + (cc * (2 - cc)) * C[idx0]);
       }
     }
 
@@ -226,30 +255,30 @@ double CMAES<SelectionPolicyType>::Optimize(
     {
       for (size_t j = 0; j < mu; ++j)
       {
-        C.slice(idx1) = C.slice(idx1) + cmu * w(j) *
-            pStep.slice(idx(j)) * pStep.slice(idx(j)).t();
+        C[idx1] = C[idx1] + cmu * w(j) *
+            pStep[idx(j)] * pStep[idx(j)].t();
       }
     }
     else
     {
       for (size_t j = 0; j < mu; ++j)
       {
-        C.slice(idx1) = C.slice(idx1) + cmu * w(j) *
-            pStep.slice(idx(j)).t() * pStep.slice(idx(j));
+        C[idx1] = C[idx1] + cmu * w(j) *
+            pStep[idx(j)].t() * pStep[idx(j)];
       }
     }
 
-    arma::eig_sym(eigval, eigvec, C.slice(idx1));
-    const arma::uvec negativeEigval = find(eigval < 0, 1);
+    arma::eig_sym(eigval, eigvec, C[idx1]);
+    const arma::uvec negativeEigval = arma::find(eigval < 0, 1);
     if (!negativeEigval.is_empty())
     {
       if (negativeEigval(0) == 0)
       {
-        C.slice(idx1).zeros();
+        C[idx1].zeros();
       }
       else
       {
-        C.slice(idx1) = eigvec.cols(0, negativeEigval(0) - 1) *
+        C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) *
             arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) *
             eigvec.cols(0, negativeEigval(0) - 1).t();
       }
@@ -263,6 +292,8 @@ double CMAES<SelectionPolicyType>::Optimize(
     {
       Warn << "CMA-ES: converged to " << overallObjective << "; "
           << "terminating with failure.  Try a smaller step size?" << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -270,12 +301,15 @@ double CMAES<SelectionPolicyType>::Optimize(
     {
       Info << "CMA-ES: minimized within tolerance " << tolerance << "; "
           << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
     lastObjective = overallObjective;
   }
 
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/cmaes/full_selection.hpp

@@ -23,24 +23,29 @@ class FullSelection
   /**
    * Select the full dataset to calculate the objective function.
    *
-   * @tparam DecomposableFunctionType Type of the function to be evaluated.
+   * @tparam SeparableFunctionType Type of the function to be evaluated.
    * @param function Function to optimize.
    * @param batchSize Batch size to use for each step.
    * @param iterate starting point.
    */
-  template<typename DecomposableFunctionType>
-  double Select(DecomposableFunctionType& function,
-                      const size_t batchSize,
-                      const arma::mat& iterate)
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  double Select(SeparableFunctionType& function,
+                const size_t batchSize,
+                const MatType& iterate,
+                CallbackTypes&... callbacks)
   {
     // Find the number of functions to use.
     const size_t numFunctions = function.NumFunctions();
 
-    double objective = 0;
+    typename MatType::elem_type objective = 0;
     for (size_t f = 0; f < numFunctions; f += batchSize)
     {
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
       objective += function.Evaluate(iterate, f, effectiveBatchSize);
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
     }
 
     return objective;

include/ensmallen_bits/cmaes/random_selection.hpp

@@ -38,20 +38,23 @@ class RandomSelection
   /**
    * Randomly select dataset points to calculate the objective function.
    *
-   * @tparam DecomposableFunctionType Type of the function to be evaluated.
+   * @tparam SeparableFunctionType Type of the function to be evaluated.
    * @param function Function to optimize.
    * @param batchSize Batch size to use for each step.
    * @param iterate starting point.
    */
-  template<typename DecomposableFunctionType>
-  double Select(DecomposableFunctionType& function,
-                      const size_t batchSize,
-                      const arma::mat& iterate)
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  double Select(SeparableFunctionType& function,
+                const size_t batchSize,
+                const MatType& iterate,
+                CallbackTypes&... callbacks)
   {
     // Find the number of functions to use.
     const size_t numFunctions = function.NumFunctions();
 
-    double objective = 0;
+    typename MatType::elem_type objective = 0;
     for (size_t f = 0; f < std::floor(numFunctions * fraction); f += batchSize)
     {
       const size_t selection = arma::as_scalar(arma::randi<arma::uvec>(
@@ -60,6 +63,8 @@ class RandomSelection
           numFunctions - selection);
 
       objective += function.Evaluate(iterate, selection, effectiveBatchSize);
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
     }
 
     return objective;

include/ensmallen_bits/cne/cne.hpp

@@ -56,8 +56,10 @@ namespace ens {
  * The whole process then repeats for multiple generation until at least one of
  * the termination criteria is met:
  *
- * 1) The final value of the objective function (Not considered if not provided).
- * 2) The maximum number of generation reached (optional but highly recommended).
+ * 1) The final value of the objective function (Not considered if not
+ *    provided).
+ * 2) The maximum number of generation reached (optional but highly
+ *    recommended).
  * 3) Minimum change in best fitness values between two consecutive generations
  *    should be greater than a threshold value (Not considered if not provided).
  *
@@ -87,30 +89,33 @@ class CNE
    *     the next generation.
    * @param tolerance The final value of the objective function for termination.
    *     If set to negative value, tolerance is not considered.
-   * @param objectiveChange Minimum change in best fitness values between two
-   *     consecutive generations should be greater than threshold. If set to
-   *     negative value, objectiveChange is not considered.
    */
   CNE(const size_t populationSize = 500,
       const size_t maxGenerations = 5000,
       const double mutationProb = 0.1,
       const double mutationSize = 0.02,
       const double selectPercent = 0.2,
-      const double tolerance = 1e-5,
-      const double objectiveChange = 1e-5);
+      const double tolerance = 1e-5);
 
   /**
    * Optimize the given function using CNE. The given
    * starting point will be modified to store the finishing point of the
    * algorithm, and the final objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam ArbitraryFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename ArbitraryFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(ArbitraryFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
 
   //! Get the population size.
   size_t PopulationSize() const { return populationSize; }
@@ -137,22 +142,21 @@ class CNE
   //! Modify the selection percentage.
   double& SelectionPercentage() { return selectPercent; }
 
-  //! Get the final objective value.
+  //! Get the tolerance.
   double Tolerance() const { return tolerance; }
-  //! Modify the final objective value.
+  //! Modify the tolerance.
   double& Tolerance() { return tolerance; }
 
-  //! Get the change in fitness history between generations.
-  double ObjectiveChange() const { return objectiveChange; }
-  //! Modify the termination criteria of change in fitness value.
-  double& ObjectiveChange() { return objectiveChange; }
-
  private:
   //! Reproduce candidates to create the next generation.
-  void Reproduce();
+  template<typename MatType>
+  void Reproduce(std::vector<MatType>& population,
+                 const MatType& fitnessValues,
+                 arma::uvec& index);
 
   //! Modify weights with some noise for the evolution of next generation.
-  void Mutate();
+  template<typename MatType>
+  void Mutate(std::vector<MatType>& population, arma::uvec& index);
 
   /**
    * Crossover parents and create new childs. Two parents create two new childs.
@@ -166,20 +170,13 @@ class CNE
    *                 generation and place a child over there for the
    *                 next generation.
    */
-  void Crossover(const size_t mom,
+  template<typename MatType>
+  void Crossover(std::vector<MatType>& population,
+                 const size_t mom,
                  const size_t dad,
                  const size_t dropout1,
                  const size_t dropout2);
 
-  //! Population matrix. Each column is a candidate.
-  arma::cube population;
-
-  //! Vector of fintness values corresponding to each candidate.
-  arma::vec fitnessValues;
-
-  //! Index of sorted fitness values.
-  arma::uvec index;
-
   //! The number of candidates in the population.
   size_t populationSize;
 
@@ -198,9 +195,6 @@ class CNE
   //! The final value of the objective function.
   double tolerance;
 
-  //! Minimum change in best fitness values between two generations.
-  double objectiveChange;
-
   //! Number of candidates to become parent for the next generation.
   size_t numElite;
 

include/ensmallen_bits/cne/cne_impl.hpp

@@ -25,23 +25,39 @@ inline CNE::CNE(const size_t populationSize,
                 const double mutationProb,
                 const double mutationSize,
                 const double selectPercent,
-                const double tolerance,
-                const double objectiveChange) :
+                const double tolerance) :
     populationSize(populationSize),
     maxGenerations(maxGenerations),
     mutationProb(mutationProb),
     mutationSize(mutationSize),
     selectPercent(selectPercent),
     tolerance(tolerance),
-    objectiveChange(objectiveChange),
     numElite(0),
     elements(0)
 { /* Nothing to do here. */ }
 
 //! Optimize the function.
-template<typename DecomposableFunctionType>
-double CNE::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+template<typename ArbitraryFunctionType,
+         typename MatType,
+         typename... CallbackTypes>
+typename MatType::elem_type CNE::Optimize(ArbitraryFunctionType& function,
+                                          MatType& iterateIn,
+                                          CallbackTypes&&... callbacks)
 {
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<ArbitraryFunctionType,
+      BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  // Vector of fitness values corresponding to each candidate.
+  BaseMatType fitnessValues;
+  //! Index of sorted fitness values.
+  arma::uvec index;
+
   // Make sure for evolution to work at least four candidates are present.
   if (populationSize < 4)
   {
@@ -70,54 +86,64 @@ double CNE::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
         "children. Increase population size.");
   }
 
-  // Set the population size and fill random values [0,1].
-  population = arma::randu(iterate.n_rows, iterate.n_cols, populationSize);
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
-  // Store the number of elements in a cube slice or a matrix column.
-  elements = population.n_rows * population.n_cols;
+  // Generate the population based on a Gaussian distribution around the given
+  // starting point.
+  std::vector<BaseMatType> population;
+  for (size_t i = 0 ; i < populationSize; ++i)
+  {
+    population.push_back(arma::randu<BaseMatType>(iterate.n_rows,
+        iterate.n_cols) + iterate);
+  }
 
-  // initializing helper variables.
+  // Store the number of elements in the objective matrix.
+  elements = iterate.n_rows * iterate.n_cols;
+
+  // Initialize helper variables.
   fitnessValues.set_size(populationSize);
 
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
   Info << "CNE initialized successfully. Optimization started."
       << std::endl;
 
   // Find the fitness before optimization using given iterate parameters.
-  size_t lastBestFitness = function.Evaluate(iterate);
+  ElemType lastBestFitness = function.Evaluate(iterate);
+  Callback::Evaluate(*this, function, iterate, lastBestFitness, callbacks...);
 
   // Iterate until maximum number of generations is obtained.
-  for (size_t gen = 1; gen <= maxGenerations; gen++)
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t gen = 1; gen <= maxGenerations && !terminate; gen++)
   {
     // Calculating fitness values of all candidates.
     for (size_t i = 0; i < populationSize; i++)
     {
        // Select a candidate and insert the parameters in the function.
-       iterate = population.slice(i);
+       iterate = population[i];
+       terminate |= Callback::StepTaken(*this, function, iterate,
+          callbacks...);
 
        // Find fitness of candidate.
        fitnessValues[i] = function.Evaluate(iterate);
+
+       Callback::Evaluate(*this, function, iterate, fitnessValues[i],
+          callbacks...);
     }
 
     Info << "Generation number: " << gen << " best fitness = "
         << fitnessValues.min() << std::endl;
 
     // Create next generation of species.
-    Reproduce();
+    Reproduce(population, fitnessValues, index);
 
     // Check for termination criteria.
-    if (tolerance >= fitnessValues.min())
+    if (std::abs(lastBestFitness - fitnessValues.min()) < tolerance)
     {
-      Info << "CNE::Optimize(): terminating. Given fitness criteria "
-          << tolerance << " > " << fitnessValues.min() << "." << std::endl;
-      break;
-    }
-
-    // Check for termination criteria.
-    if (lastBestFitness - fitnessValues.min() < objectiveChange)
-    {
-      Info << "CNE::Optimize(): terminating. Fitness history change "
-          << (lastBestFitness - fitnessValues.min())
-          << " < " << objectiveChange << "." << std::endl;
+      Info << "CNE: minimized within tolerance " << tolerance << "; "
+            << "terminating optimization." << std::endl;
       break;
     }
 
@@ -126,13 +152,20 @@ double CNE::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
   }
 
   // Set the best candidate into the network parameters.
-  iterate = population.slice(index(0));
+  iterateIn = population[index(0)];
 
-  return function.Evaluate(iterate);
+  const ElemType objective = function.Evaluate(iterate);
+  Callback::Evaluate(*this, function, iterate, objective, callbacks...);
+
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
+  return objective;
 }
 
 //! Reproduce candidates to create the next generation.
-inline void CNE::Reproduce()
+template<typename MatType>
+inline void CNE::Reproduce(std::vector<MatType>& population,
+                           const MatType& fitnessValues,
+                           arma::uvec& index)
 {
   // Sort fitness values. Smaller fitness value means better performance.
   index = arma::sort_index(fitnessValues);
@@ -167,54 +200,56 @@ inline void CNE::Reproduce()
 
     // Parents generate 2 children replacing the dropped-out candidates.
     // Also finding the index of these candidates in the population matrix.
-    Crossover(index[mom], index[dad], index[i], index[i + 1]);
+    Crossover(population, index[mom], index[dad], index[i], index[i + 1]);
   }
 
   // Mutating the weights with small noise values.
   // This is done to bring change in the next generation.
-  Mutate();
+  Mutate(population, index);
 }
 
 //! Crossover parents to create new children.
-inline void CNE::Crossover(const size_t mom,
+template<typename MatType>
+inline void CNE::Crossover(std::vector<MatType>& population,
+                           const size_t mom,
                            const size_t dad,
                            const size_t child1,
                            const size_t child2)
 {
   // Replace the candidates with parents at their place.
-  population.slice(child1) = population.slice(mom);
-  population.slice(child2) = population.slice(dad);
-
-  // Preallocate random selection vector (values between 0 and 1).
-  arma::vec selection = arma::randu(elements);
+  population[child1] = population[mom];
+  population[child2] = population[dad];
 
   // Randomly alter mom and dad genome weights to get two different children.
   for (size_t i = 0; i < elements; i++)
   {
     // Using it to alter the weights of the children.
-    if (selection(i) > 0.5)
+    const double random = arma::randu<typename MatType::elem_type>();
+    if (random > 0.5)
     {
-      population.slice(child1)(i) = population.slice(mom)(i);
-      population.slice(child2)(i) = population.slice(dad)(i);
+      population[child1](i) = population[mom](i);
+      population[child2](i) = population[dad](i);
     }
     else
     {
-      population.slice(child1)(i) = population.slice(dad)(i);
-      population.slice(child2)(i) = population.slice(mom)(i);
+      population[child1](i) = population[dad](i);
+      population[child2](i) = population[mom](i);
     }
   }
 }
 
 //! Modify weights with some noise for the evolution of next generation.
-inline void CNE::Mutate()
+template<typename MatType>
+inline void CNE::Mutate(std::vector<MatType>& population, arma::uvec& index)
 {
   // Mutate the whole matrix with the given rate and probability.
   // The best candidate is not altered.
   for (size_t i = 1; i < populationSize; i++)
   {
-    population.slice(index(i)) += (arma::randu(
-        population.n_rows, population.n_cols) < mutationProb) %
-        (mutationSize * arma::randn(population.n_rows, population.n_cols));
+    population[index(i)] += (arma::randu<MatType>(population[index(i)].n_rows,
+        population[index(i)].n_cols) < mutationProb) %
+        (mutationSize * arma::randn<MatType>(population[index(i)].n_rows,
+        population[index(i)].n_cols));
   }
 }
 

include/ensmallen_bits/config.hpp

@@ -46,7 +46,28 @@
 #if defined(ENS_USE_OPENMP)
   #define ENS_PRAGMA_OMP_PARALLEL _Pragma("omp parallel")
   #define ENS_PRAGMA_OMP_ATOMIC   _Pragma("omp atomic")
+  #define ENS_PRAGMA_OMP_CRITICAL _Pragma("omp critical")
+  #define ENS_PRAGMA_OMP_CRITICAL_NAMED _Pragma("omp critical(section)")
 #else
   #define ENS_PRAGMA_OMP_PARALLEL
   #define ENS_PRAGMA_OMP_ATOMIC
+  #define ENS_PRAGMA_OMP_CRITICAL
+  #define ENS_PRAGMA_OMP_CRITICAL_NAMED
+#endif
+
+
+// Define ens_deprecated for deprecated functionality.
+// This is adapted from Armadillo's implementation.
+#if defined(_MSC_VER)
+  #define ens_deprecated __declspec(deprecated)
+#elif defined(__GNUG__) && (!defined(__clang__))
+  #define ens_deprecated __attribute__((__deprecated__))
+#elif defined(__clang__)
+  #if __has_attribute(__deprecated__)
+    #define ens_deprecated __attribute__((__deprecated__))
+  #else
+    #define ens_deprecated
+  #endif
+#else
+  #define ens_deprecated
 #endif

include/ensmallen_bits/de/de.hpp

@@ -0,0 +1,150 @@
+/**
+ * @file de.hpp
+ * @author Rahul Ganesh Prabhu
+ *
+ * Differential Evolution is a method used for global optimization of arbitrary
+ * functions that optimizes a problem by iteratively trying to improve a
+ * candidate solution.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef ENSMALLEN_DE_DE_HPP
+#define ENSMALLEN_DE_DE_HPP
+
+namespace ens {
+
+/**
+ * Differential evolution is a stochastic evolutionary algorithm used for global
+ * optimization. This class implements the best/1/bin strategy of differential
+ * evolution to converge a given function to minima.
+ *
+ * The algorithm works by generating a fixed number of candidates from the
+ * given starting point. At each pass through the population, the algorithm
+ * mutates each candidate solution to create a trial solution. If the trial
+ * solution is better than the candidate, it is replaced in the
+ * population.
+ *
+ * The evolution takes place in two steps:
+ * - Mutation
+ * - Crossover
+ *
+ * Mutation is done by generating a new candidate solution from the best
+ * candidate of the previous solution and two random other candidates.
+ *
+ * Crossover is done by mixing the parameters of the candidate solution and the
+ * mutant solution. This is done only if a randomly generated number between 0
+ * and 1 is greater than the crossover rate.
+ *
+ * The final value and the parameters are returned by the Optimize() method.
+ *
+ * For more information, see the following:
+ *
+ * @code
+ * @techreport{storn1995,
+ *   title    = {Differential Evolution—a simple and efficient adaptive scheme
+ *               for global optimization over continuous spaces},
+ *   author   = {Storn, Rainer and Price, Kenneth},
+ *   year     = 1995
+ * }
+ * @endcode
+ *
+ * DE can optimize arbitrary functions.  For more details, see the
+ * documentation on function types included with this distribution or on the
+ * ensmallen website.
+ */
+class DE
+{
+ public:
+  /**
+   * Constructor for the DE optimizer
+   *
+   * The default values provided over here are not necessarily suitable for a
+   * given function. Therefore it is highly recommended to adjust the
+   * parameters according to the problem.
+   *
+   * @param populationSize The number of candidates in the population.
+   *     This should be at least 3 in size.
+   * @param maxGenerations The maximum number of generations allowed for CNE.
+   * @param crossoverRate  The probability that a crossover will occur.
+   * @param differentialWeight A parameter used in the mutation of candidate
+   *     solutions controls amplification factor of the differentiation.
+   * @param tolerance The final value of the objective function for termination.
+   */
+  DE(const size_t populationSize = 100,
+     const size_t maxGenerations = 2000,
+     const double crossoverRate = 0.6,
+     const double differentialWeight = 0.8,
+     const double tolerance = 1e-5);
+
+  /**
+   * Optimize the given function using DE. The given
+   * starting point will be modified to store the finishing point of the
+   * algorithm, and the final objective value is returned.
+   *
+   * @tparam FunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize.
+   * @tparam CallbackTypes Types of callback functions.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
+   * @return Objective value of the final point.
+   */
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Get the population size.
+  size_t PopulationSize() const { return populationSize; }
+  //! Modify the population size.
+  size_t& PopulationSize() { return populationSize; }
+
+  //! Get maximum number of generations.
+  size_t MaxGenerations() const { return maxGenerations; }
+  //! Modify maximum number of generations.
+  size_t& MaxGenerations() { return maxGenerations; }
+
+  //! Get crossover rate.
+  double CrossoverRate() const { return crossoverRate; }
+  //! Modify crossover rate.
+  double& CrossoverRate() { return crossoverRate; }
+
+  //! Get differential weight.
+  double DifferentialWeight() const {return differentialWeight; }
+  //! Modify differential weight.
+  double& DifferentialWeight() { return differentialWeight; }
+
+  //! Get the tolerance.
+  double Tolerance() const { return tolerance; }
+  //! Modify the tolerance.
+  double& Tolerance() { return tolerance; }
+
+ private:
+  //! The number of candidates in the population.
+  size_t populationSize;
+
+  //! Maximum number of generations before termination criteria is met.
+  size_t maxGenerations;
+
+  //! Probability that crossover will occur.
+  double crossoverRate;
+
+  //! Amplification factor for differentiation.
+  double differentialWeight;
+
+  //! The tolerance for termination.
+  double tolerance;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "de_impl.hpp"
+
+#endif

include/ensmallen_bits/de/de_impl.hpp

@@ -0,0 +1,176 @@
+/**
+ * @file de_impl.hpp
+ * @author Rahul Ganesh Prabhu
+ *
+ * Implementation of Differential Evolution an evolutionary algorithm used for
+ * global optimization of arbitrary functions.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_DE_DE_IMPL_HPP
+#define ENSMALLEN_DE_DE_IMPL_HPP
+
+#include "de.hpp"
+
+namespace ens {
+
+inline DE::DE(const size_t populationSize ,
+              const size_t maxGenerations,
+              const double crossoverRate,
+              const double differentialWeight,
+              const double tolerance):
+    populationSize(populationSize),
+    maxGenerations(maxGenerations),
+    crossoverRate(crossoverRate),
+    differentialWeight(differentialWeight),
+    tolerance(tolerance)
+{ /* Nothing to do here. */ }
+
+//!Optimize the function
+template<typename FunctionType,
+         typename MatType,
+         typename... CallbackTypes>
+typename MatType::elem_type DE::Optimize(FunctionType& function,
+                                         MatType& iterateIn,
+                                         CallbackTypes&&... callbacks)
+{
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
+  // Population matrix. Each column is a candidate.
+  std::vector<BaseMatType> population;
+  population.resize(populationSize);
+  // Vector of fitness values corresponding to each candidate.
+  arma::Col<ElemType> fitnessValues;
+
+  // Make sure that we have the methods that we need.  Long name...
+  traits::CheckArbitraryFunctionTypeAPI<
+      FunctionType, BaseMatType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+
+  // Population Size must be at least 3 for DE to work.
+  if (populationSize < 3)
+  {
+    throw std::logic_error("CNE::Optimize(): population size should be at least"
+        " 3!");
+  }
+
+  // Initialize helper variables.
+  fitnessValues.set_size(populationSize);
+  ElemType lastBestFitness = DBL_MAX;
+  BaseMatType bestElement;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  // Generate a population based on a Gaussian distribution around the given
+  // starting point. Also finds the best element of the population.
+  for (size_t i = 0; i < populationSize; i++)
+  {
+    population[i].randn(iterate.n_rows, iterate.n_cols);
+    population[i] += iterate;
+    fitnessValues[i] = function.Evaluate(population[i]);
+
+    Callback::Evaluate(*this, function, population[i], fitnessValues[i],
+        callbacks...);
+
+    if (fitnessValues[i] < lastBestFitness)
+    {
+      lastBestFitness = fitnessValues[i];
+      bestElement = population[i];
+    }
+  }
+
+  // Iterate until maximum number of generations are completed.
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t gen = 0; gen < maxGenerations && !terminate; gen++)
+  {
+    // Generate new population based on /best/1/bin strategy.
+    for (size_t member = 0; member < populationSize; member++)
+    {
+      iterate = population[member];
+
+      // Generate two different random numbers to choose two random members.
+      size_t l = 0, m = 0;
+      do
+      {
+        l = arma::randi<arma::uword>(arma::distr_param(0, populationSize - 1));
+      }
+      while (l == member);
+
+      do
+      {
+        m = arma::randi<arma::uword>(arma::distr_param(0, populationSize - 1));
+      }
+      while (m == member && m == l);
+
+      // Generate new "mutant" from two randomly chosen members.
+      BaseMatType mutant = bestElement + differentialWeight *
+          (population[l] - population[m]);
+
+      // Perform crossover.
+      const BaseMatType cr = arma::randu<BaseMatType>(iterate.n_rows);
+      for (size_t it = 0; it < iterate.n_rows; it++)
+      {
+        if (cr[it] >= crossoverRate)
+        {
+          mutant[it] = iterate[it];
+        }
+      }
+
+      ElemType iterateValue = function.Evaluate(iterate);
+      Callback::Evaluate(*this, function, iterate, iterateValue, callbacks...);
+
+      const ElemType mutantValue = function.Evaluate(mutant);
+      Callback::Evaluate(*this, function, mutant, mutantValue, callbacks...);
+
+      // Replace the current member if mutant is better.
+      if (mutantValue < iterateValue)
+      {
+        iterate = mutant;
+        iterateValue = mutantValue;
+
+        terminate |= Callback::StepTaken(*this, function, iterate,
+            callbacks...);
+      }
+
+      fitnessValues[member] = iterateValue;
+      population[member] = iterate;
+    }
+
+    // Check for termination criteria.
+    if (std::abs(lastBestFitness - fitnessValues.min()) < tolerance)
+    {
+      Info << "DE: minimized within tolerance " << tolerance << "; "
+          << "terminating optimization." << std::endl;
+      break;
+    }
+
+    // Update helper variables.
+    lastBestFitness = fitnessValues.min();
+    for (size_t it = 0; it < populationSize; it++)
+    {
+      if (fitnessValues[it] == lastBestFitness)
+      {
+        bestElement = population[it];
+        break;
+      }
+    }
+  }
+
+  iterate = bestElement;
+
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
+  return lastBestFitness;
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/ens_version.hpp

@@ -12,16 +12,16 @@
 
 // This follows the Semantic Versioning pattern defined in https://semver.org/.
 
-#define ENS_VERSION_MAJOR 1
+#define ENS_VERSION_MAJOR 2
 // The minor version is two digits so regular numerical comparisons of versions
 // work right.  The first minor version of a release is always 10.
-#define ENS_VERSION_MINOR 13
-#define ENS_VERSION_PATCH 0
+#define ENS_VERSION_MINOR 14
+#define ENS_VERSION_PATCH 2
 // If this is a release candidate, it will be reflected in the version name
 // (i.e. the version name will be "RC1", "RC2", etc.).  Otherwise the version
 // name will typically be a seemingly arbitrary set of words that does not
 // contain the capitalized string "RC".
-#define ENS_VERSION_NAME "Coronavirus Invasion"
+#define ENS_VERSION_NAME "No Direction Home"
 
 namespace ens {
 

include/ensmallen_bits/eve/eve.hpp

@@ -31,8 +31,9 @@ namespace ens {
  *   year    = {2016},
  *   url     = {http://arxiv.org/abs/1611.01505}
  * }
+ * @endcode
  *
- * For Eve to work, a DecomposableFunctionType template parameter is required.
+ * For Eve to work, a SeparableFunctionType template parameter is required.
  * This class must implement the following function:
  *
  *   size_t NumFunctions();
@@ -72,6 +73,8 @@ class Eve
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param shuffle If true, the function order is shuffled; otherwise, each
    *        function is visited in linear order.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   Eve(const double stepSize = 0.001,
       const size_t batchSize = 32,
@@ -82,20 +85,45 @@ class Eve
       const double clip = 10,
       const size_t maxIterations = 100000,
       const double tolerance = 1e-5,
-      const bool shuffle = true);
+      const bool shuffle = true,
+      const bool exactObjective = false);
 
   /**
-   * Optimize the given function using stochastic gradient descent.  The given
-   * starting point will be modified to store the finishing point of the
-   * algorithm, and the final objective value is returned.
+   * Optimize the given function using Eve. The given starting point will be
+   * modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of the parameters matrix.
+   * @tparam GradType Type of the gradient matrix.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the step size.
   double StepSize() const { return stepSize; }
@@ -147,6 +175,11 @@ class Eve
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return shuffle; }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return exactObjective; }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return exactObjective; }
+
  private:
   //! The step size for each example.
   double stepSize;
@@ -178,6 +211,9 @@ class Eve
   //! Controls whether or not the individual functions are shuffled when
   //! iterating.
   bool shuffle;
+
+  //! Controls whether or not the actual Objective value is calculated.
+  bool exactObjective;
 };
 
 } // namespace ens

include/ensmallen_bits/eve/eve_impl.hpp

@@ -29,7 +29,8 @@ inline Eve::Eve(const double stepSize,
                 const double clip,
                 const size_t maxIterations,
                 const double tolerance,
-                const bool shuffle) :
+                const bool shuffle,
+                const bool exactObjective) :
     stepSize(stepSize),
     batchSize(batchSize),
     beta1(beta1),
@@ -39,75 +40,71 @@ inline Eve::Eve(const double stepSize,
     clip(clip),
     maxIterations(maxIterations),
     tolerance(tolerance),
-    shuffle(shuffle)
+    shuffle(shuffle),
+    exactObjective(exactObjective)
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
-template<typename DecomposableFunctionType>
-double Eve::Optimize(
-    DecomposableFunctionType& function,
-    arma::mat& iterate)
+template<typename SeparableFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+Eve::Optimize(SeparableFunctionType& function,
+              MatType& iterateIn,
+              CallbackTypes&&... callbacks)
 {
-  typedef Function<DecomposableFunctionType> FullFunctionType;
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<SeparableFunctionType, BaseMatType, BaseGradType>
+      FullFunctionType;
   FullFunctionType& f(static_cast<FullFunctionType&>(function));
 
   // Make sure we have all the methods that we need.
-  traits::CheckDecomposableFunctionTypeAPI<FullFunctionType>();
+  traits::CheckSeparableFunctionTypeAPI<FullFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // Find the number of functions to use.
   const size_t numFunctions = f.NumFunctions();
 
   // To keep track of where we are and how things are going.
   size_t currentFunction = 0;
-  double overallObjective = 0;
-  double lastOverallObjective = DBL_MAX;
+  size_t epoch = 1;
+  ElemType overallObjective = 0;
+  ElemType lastOverallObjective = DBL_MAX;
 
-  double objective = 0;
-  double lastObjective = 0;
-  double dt = 1;
+  ElemType objective = 0;
+  ElemType lastObjective = 0;
+  ElemType dt = 1;
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
   // The exponential moving average of gradient values.
-  arma::mat m = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
+  BaseGradType m(iterate.n_rows, iterate.n_cols);
+  m.zeros();
 
   // The exponential moving average of squared gradient values.
-  arma::mat v = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
+  BaseGradType v(iterate.n_rows, iterate.n_cols);
+  v.zeros();
 
   // Now iterate!
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
   const size_t actualMaxIterations = (maxIterations == 0) ?
       std::numeric_limits<size_t>::max() : maxIterations;
-  for (size_t i = 0; i < actualMaxIterations; /* incrementing done manually */)
+  for (size_t i = 0; i < actualMaxIterations && !terminate;
+      /* incrementing done manually */)
   {
-    // Is this iteration the start of a sequence?
-    if ((currentFunction % numFunctions) == 0 && i > 0)
-    {
-      // Output current objective function.
-      Info << "Eve: iteration " << i << ", objective " << overallObjective
-          << "." << std::endl;
-
-      if (std::isnan(overallObjective) || std::isinf(overallObjective))
-      {
-        Warn << "Eve: converged to " << overallObjective << "; terminating"
-            << " with failure.  Try a smaller step size?" << std::endl;
-        return overallObjective;
-      }
-
-      if (std::abs(lastOverallObjective - overallObjective) < tolerance)
-      {
-        Info << "Eve: minimized within tolerance " << tolerance << "; "
-            << "terminating optimization." << std::endl;
-        return overallObjective;
-      }
-
-      // Reset the counter variables.
-      lastOverallObjective = overallObjective;
-      overallObjective = 0;
-      currentFunction = 0;
-
-      if (shuffle) // Determine order of visitation.
-        f.Shuffle();
-    }
-
     // Find the effective batch size; we have to take the minimum of three
     // things:
     // - the batch size can't be larger than the user-specified batch size;
@@ -124,6 +121,9 @@ double Eve::Optimize(
         gradient, effectiveBatchSize);
     overallObjective += objective;
 
+    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        objective, gradient, callbacks...);
+
     m *= beta1;
     m += (1 - beta1) * gradient;
 
@@ -135,11 +135,12 @@ double Eve::Optimize(
 
     if (i > 0)
     {
-      const double d = std::abs(objective - lastObjective) /
+      const ElemType d = std::abs(objective - lastObjective) /
           (std::min(objective, lastObjective) + epsilon);
 
       dt *= beta3;
-      dt += (1 - beta3) * std::min(std::max(d, 1.0 / clip), clip);
+      dt += (1 - beta3) * std::min(std::max(d, ElemType(1.0 / clip)),
+          ElemType(clip));
     }
 
     lastObjective = objective;
@@ -147,20 +148,69 @@ double Eve::Optimize(
     iterate -= stepSize / dt * (m / biasCorrection1) /
         (arma::sqrt(v / biasCorrection2) + epsilon);
 
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
+
     i += effectiveBatchSize;
     currentFunction += effectiveBatchSize;
+
+    // Is this iteration the start of a sequence?
+    if ((currentFunction % numFunctions) == 0)
+    {
+      terminate |= Callback::EndEpoch(*this, f, iterate, epoch++,
+          overallObjective / (ElemType) numFunctions, callbacks...);
+
+      // Output current objective function.
+      Info << "Eve: iteration " << i << ", objective " << overallObjective
+          << "." << std::endl;
+
+      if (std::isnan(overallObjective) || std::isinf(overallObjective))
+      {
+        Warn << "Eve: converged to " << overallObjective << "; terminating"
+            << " with failure.  Try a smaller step size?" << std::endl;
+
+        Callback::EndOptimization(*this, f, iterate, callbacks...);
+        return overallObjective;
+      }
+
+      if (std::abs(lastOverallObjective - overallObjective) < tolerance ||
+          Callback::BeginEpoch(*this, f, iterate, epoch, overallObjective,
+              callbacks...))
+      {
+        Info << "Eve: minimized within tolerance " << tolerance << "; "
+            << "terminating optimization." << std::endl;
+
+        Callback::EndOptimization(*this, f, iterate, callbacks...);
+        return overallObjective;
+      }
+
+      // Reset the counter variables.
+      lastOverallObjective = overallObjective;
+      overallObjective = 0;
+      currentFunction = 0;
+
+      if (shuffle) // Determine order of visitation.
+        f.Shuffle();
+    }
   }
 
   Info << "Eve: maximum iterations (" << maxIterations << ") reached; "
       << "terminating optimization." << std::endl;
 
-  // Calculate final objective.
-  overallObjective = 0;
-  for (size_t i = 0; i < numFunctions; i += batchSize)
+  // Calculate final objective if exactObjective is set to true.
+  if (exactObjective)
   {
-    const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
-    overallObjective += f.Evaluate(iterate, i, effectiveBatchSize);
+    overallObjective = 0;
+    for (size_t i = 0; i < numFunctions; i += batchSize)
+    {
+      const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
+      const ElemType objective = f.Evaluate(iterate, i, effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, f, iterate, objective, callbacks...);
+    }
   }
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return overallObjective;
 }
 

include/ensmallen_bits/ftml/ftml.hpp

@@ -66,6 +66,8 @@ class FTML
    *        function is visited in linear order.
    * @param resetPolicy If true, parameters are reset before every Optimize
    *        call; otherwise, their values are retained.
+   * @param exactObjective Calculate the exact objective (Default: estimate the
+   *        final objective obtained on the last pass over the data).
    */
   FTML(const double stepSize = 0.001,
        const size_t batchSize = 32,
@@ -75,22 +77,49 @@ class FTML
        const size_t maxIterations = 100000,
        const double tolerance = 1e-5,
        const bool shuffle = true,
-       const bool resetPolicy = true);
+       const bool resetPolicy = true,
+       const bool exactObjective = false);
 
   /**
    * Optimize the given function using FTML. The given starting point will
    * be modified to store the finishing point of the algorithm, and the final
    * objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks)
   {
-    return optimizer.Optimize(function, iterate);
+    return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
   }
 
   //! Get the step size.
@@ -133,6 +162,11 @@ class FTML
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
+  //! Get whether or not the actual objective is calculated.
+  bool ExactObjective() const { return optimizer.ExactObjective(); }
+  //! Modify whether or not the actual objective is calculated.
+  bool& ExactObjective() { return optimizer.ExactObjective(); }
+
   //! Get whether or not the update policy parameters
   //! are reset before Optimize call.
   bool ResetPolicy() const { return optimizer.ResetPolicy(); }

include/ensmallen_bits/ftml/ftml_impl.hpp

@@ -25,7 +25,8 @@ inline FTML::FTML(const double stepSize,
                   const size_t maxIterations,
                   const double tolerance,
                   const bool shuffle,
-                  const bool resetPolicy) :
+                  const bool resetPolicy,
+                  const bool exactObjective) :
     optimizer(stepSize,
               batchSize,
               maxIterations,
@@ -33,7 +34,8 @@ inline FTML::FTML(const double stepSize,
               shuffle,
               FTMLUpdate(epsilon, beta1, beta2),
               NoDecay(),
-              resetPolicy)
+              resetPolicy,
+              exactObjective)
 { /* Nothing to do. */ }
 
 } // namespace ens

include/ensmallen_bits/ftml/ftml_update.hpp

@@ -34,7 +34,6 @@ namespace ens {
  * }
  * @endcode
  */
-
 class FTMLUpdate
 {
  public:
@@ -44,7 +43,7 @@ class FTMLUpdate
    * @param epsilon Epsilon is the minimum allowed gradient.
    * @param beta1 Exponential decay rate for the first moment estimates.
    * @param beta2 Exponential decay rate for the weighted infinity norm
-            estimates.
+   *        estimates.
    */
   FTMLUpdate(const double epsilon = 1e-8,
              const double beta1 = 0.9,
@@ -55,51 +54,6 @@ class FTMLUpdate
       iteration(0)
   { /* Do nothing. */ }
 
-  /**
-   * The Initialize method is called by SGD::Optimize method with UpdatePolicy
-   * FTMLUpdate before the start of the iteration update process.
-   *
-   * @param rows Number of rows in the gradient matrix.
-   * @param cols Number of columns in the gradient matrix.
-   */
-  void Initialize(const size_t rows, const size_t cols)
-  {
-    v = arma::zeros<arma::mat>(rows, cols);
-    z = arma::zeros<arma::mat>(rows, cols);
-    d = arma::zeros<arma::mat>(rows, cols);
-  }
-
-  /**
-   * Update step for FTML.
-   *
-   * @param iterate Parameter that minimizes the function.
-   * @param stepSize Step size to be used for the given iteration.
-   * @param gradient The gradient matrix.
-   */
-  void Update(arma::mat& iterate,
-              const double stepSize,
-              const arma::mat& gradient)
-  {
-    // Increment the iteration counter variable.
-    ++iteration;
-
-    // And update the iterate.
-    v *= beta2;
-    v += (1 - beta2) * (gradient % gradient);
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
-
-    arma::mat sigma = -beta1 * d;
-    d = biasCorrection1 / stepSize *
-      (arma::sqrt(v / biasCorrection2) + epsilon);
-    sigma += d;
-
-    z *= beta1;
-    z += (1 - beta1) * gradient - sigma % iterate;
-    iterate = -z / d;
-  }
-
   //! Get the value used to initialise the squared gradient parameter.
   double Epsilon() const { return epsilon; }
   //! Modify the value used to initialise the squared gradient parameter.
@@ -115,6 +69,84 @@ class FTMLUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the current iteration number.
+  size_t Iteration() const { return iteration; }
+  //! Modify the current iteration number.
+  size_t& Iteration() { return iteration; }
+
+  /**
+   * The UpdatePolicyType policy classes must contain an internal 'Policy'
+   * template class with two template arguments: MatType and GradType.  This is
+   * instantiated at the start of the optimization, and holds parameters
+   * specific to an individual optimization.
+   */
+  template<typename MatType, typename GradType>
+  class Policy
+  {
+   public:
+    /**
+     * This constructor is called by the SGD Optimize() method before the start
+     * of the iteration update process.
+     *
+     * @param parent AdamUpdate object.
+     * @param rows Number of rows in the gradient matrix.
+     * @param cols Number of columns in the gradient matrix.
+     */
+    Policy(FTMLUpdate& parent, const size_t rows, const size_t cols) :
+        parent(parent)
+    {
+      v.zeros(rows, cols);
+      z.zeros(rows, cols);
+      d.zeros(rows, cols);
+    }
+
+    /**
+     * Update step for FTML.
+     *
+     * @param iterate Parameter that minimizes the function.
+     * @param stepSize Step size to be used for the given iteration.
+     * @param gradient The gradient matrix.
+     */
+    void Update(MatType& iterate,
+                const double stepSize,
+                const GradType& gradient)
+    {
+      // Increment the iteration counter variable.
+      ++parent.iteration;
+
+      // And update the iterate.
+      v *= parent.beta2;
+      v += (1 - parent.beta2) * (gradient % gradient);
+
+      const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
+          parent.iteration);
+      const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
+          parent.iteration);
+
+      MatType sigma = -parent.beta1 * d;
+      d = biasCorrection1 / stepSize *
+        (arma::sqrt(v / biasCorrection2) + parent.epsilon);
+      sigma += d;
+
+      z *= parent.beta1;
+      z += (1 - parent.beta1) * gradient - sigma % iterate;
+      iterate = -z / d;
+    }
+
+   private:
+    // Reference to instantiated parent object.
+    FTMLUpdate& parent;
+
+    // The exponential moving average of gradient values.
+    GradType v;
+
+    // The exponential moving average of squared gradient values.
+    GradType z;
+
+    // Parameter update term.
+    MatType d;
+  };
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -125,17 +157,8 @@ class FTMLUpdate
   // The second moment coefficient.
   double beta2;
 
-  // The exponential moving average of gradient values.
-  arma::mat v;
-
-  // The exponential moving average of squared gradient values.
-  arma::mat z;
-
-  // Parmeter update term.
-  arma::mat d;
-
   // The number of iterations.
-  double iteration;
+  size_t iteration;
 };
 
 } // namespace ens

include/ensmallen_bits/function.hpp

@@ -15,7 +15,7 @@
 
 namespace ens {
 
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 class Function;
 
 } // namespace ens
@@ -25,9 +25,9 @@ class Function;
 #include "function/add_evaluate.hpp"
 #include "function/add_gradient.hpp"
 #include "function/add_evaluate_with_gradient.hpp"
-#include "function/add_decomposable_evaluate.hpp"
-#include "function/add_decomposable_gradient.hpp"
-#include "function/add_decomposable_evaluate_with_gradient.hpp"
+#include "function/add_separable_evaluate.hpp"
+#include "function/add_separable_gradient.hpp"
+#include "function/add_separable_evaluate_with_gradient.hpp"
 
 namespace ens {
 
@@ -54,26 +54,28 @@ namespace ens {
  * addition, this class does not (to the best of my knowledge) rely on any
  * undefined behavior.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 class Function :
-    public AddDecomposableEvaluateWithGradientStatic<FunctionType>,
-    public AddDecomposableEvaluateWithGradientConst<FunctionType>,
-    public AddDecomposableEvaluateWithGradient<FunctionType>,
-    public AddDecomposableGradientStatic<FunctionType>,
-    public AddDecomposableGradientConst<FunctionType>,
-    public AddDecomposableGradient<FunctionType>,
-    public AddDecomposableEvaluateStatic<FunctionType>,
-    public AddDecomposableEvaluateConst<FunctionType>,
-    public AddDecomposableEvaluate<FunctionType>,
-    public AddEvaluateWithGradientStatic<FunctionType>,
-    public AddEvaluateWithGradientConst<FunctionType>,
-    public AddEvaluateWithGradient<FunctionType>,
-    public AddGradientStatic<FunctionType>,
-    public AddGradientConst<FunctionType>,
-    public AddGradient<FunctionType>,
-    public AddEvaluateStatic<FunctionType>,
-    public AddEvaluateConst<FunctionType>,
-    public AddEvaluate<FunctionType>,
+    public AddSeparableEvaluateWithGradientStatic<FunctionType, MatType,
+        GradType>,
+    public AddSeparableEvaluateWithGradientConst<FunctionType, MatType,
+        GradType>,
+    public AddSeparableEvaluateWithGradient<FunctionType, MatType, GradType>,
+    public AddSeparableGradientStatic<FunctionType, MatType, GradType>,
+    public AddSeparableGradientConst<FunctionType, MatType, GradType>,
+    public AddSeparableGradient<FunctionType, MatType, GradType>,
+    public AddSeparableEvaluateStatic<FunctionType, MatType, GradType>,
+    public AddSeparableEvaluateConst<FunctionType, MatType, GradType>,
+    public AddSeparableEvaluate<FunctionType, MatType, GradType>,
+    public AddEvaluateWithGradientStatic<FunctionType, MatType, GradType>,
+    public AddEvaluateWithGradientConst<FunctionType, MatType, GradType>,
+    public AddEvaluateWithGradient<FunctionType, MatType, GradType>,
+    public AddGradientStatic<FunctionType, MatType, GradType>,
+    public AddGradientConst<FunctionType, MatType, GradType>,
+    public AddGradient<FunctionType, MatType, GradType>,
+    public AddEvaluateStatic<FunctionType, MatType, GradType>,
+    public AddEvaluateConst<FunctionType, MatType, GradType>,
+    public AddEvaluate<FunctionType, MatType, GradType>,
     public FunctionType
 {
  public:
@@ -81,26 +83,29 @@ class Function :
   // an unconstructable overload with the same name, so we can use using
   // declarations here to ensure that they are all accessible.  Since we don't
   // know what FunctionType has, we can't use any using declarations there.
-  using AddDecomposableEvaluateWithGradientStatic<
-      FunctionType>::EvaluateWithGradient;
-  using AddDecomposableEvaluateWithGradientConst<
-      FunctionType>::EvaluateWithGradient;
-  using AddDecomposableEvaluateWithGradient<FunctionType>::EvaluateWithGradient;
-  using AddDecomposableGradientStatic<FunctionType>::Gradient;
-  using AddDecomposableGradientConst<FunctionType>::Gradient;
-  using AddDecomposableGradient<FunctionType>::Gradient;
-  using AddDecomposableEvaluateStatic<FunctionType>::Evaluate;
-  using AddDecomposableEvaluateConst<FunctionType>::Evaluate;
-  using AddDecomposableEvaluate<FunctionType>::Evaluate;
-  using AddEvaluateWithGradientStatic<FunctionType>::EvaluateWithGradient;
-  using AddEvaluateWithGradientConst<FunctionType>::EvaluateWithGradient;
-  using AddEvaluateWithGradient<FunctionType>::EvaluateWithGradient;
-  using AddGradientStatic<FunctionType>::Gradient;
-  using AddGradientConst<FunctionType>::Gradient;
-  using AddGradient<FunctionType>::Gradient;
-  using AddEvaluateStatic<FunctionType>::Evaluate;
-  using AddEvaluateConst<FunctionType>::Evaluate;
-  using AddEvaluate<FunctionType>::Evaluate;
+  using AddSeparableEvaluateWithGradientStatic<
+      FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddSeparableEvaluateWithGradientConst<
+      FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddSeparableEvaluateWithGradient<
+      FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddSeparableGradientStatic<
+      FunctionType, MatType, GradType>::Gradient;
+  using AddSeparableGradientConst<FunctionType, MatType, GradType>::Gradient;
+  using AddSeparableGradient<FunctionType, MatType, GradType>::Gradient;
+  using AddSeparableEvaluateStatic<
+      FunctionType, MatType, GradType>::Evaluate;
+  using AddSeparableEvaluateConst<FunctionType, MatType, GradType>::Evaluate;
+  using AddSeparableEvaluate<FunctionType, MatType, GradType>::Evaluate;
+  using AddEvaluateWithGradientStatic<FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddEvaluateWithGradientConst<FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddEvaluateWithGradient<FunctionType, MatType, GradType>::EvaluateWithGradient;
+  using AddGradientStatic<FunctionType, MatType, GradType>::Gradient;
+  using AddGradientConst<FunctionType, MatType, GradType>::Gradient;
+  using AddGradient<FunctionType, MatType, GradType>::Gradient;
+  using AddEvaluateStatic<FunctionType, MatType, GradType>::Evaluate;
+  using AddEvaluateConst<FunctionType, MatType, GradType>::Evaluate;
+  using AddEvaluate<FunctionType, MatType, GradType>::Evaluate;
 };
 
 } // namespace ens

include/ensmallen_bits/function/add_decomposable_evaluate.hpp

@@ -1,214 +0,0 @@
-/**
- * @file add_decomposable_evaluate.hpp
- * @author Ryan Curtin
- *
- * Adds a decomposable Evaluate() function if a decomposable
- * EvaluateWithGradient() function exists.
- *
- * ensmallen is free software; you may redistribute it and/or modify it under
- * the terms of the 3-clause BSD license.  You should have received a copy of
- * the 3-clause BSD license along with ensmallen.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_HPP
-#define ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_HPP
-
-#include "traits.hpp"
-
-namespace ens {
-
-/**
- * The AddDecomposableEvaluate mixin class will add a decomposable Evaluate()
- * method if a decomposable EvaluateWithGradient() function exists, or nothing
- * otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateForm>::value>
-class AddDecomposableEvaluate
-{
- public:
-  // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&, const size_t, const size_t);
-};
-
-/**
- * Reflect the existing Evaluate().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableEvaluate<FunctionType, HasDecomposableEvaluateWithGradient,
-    true>
-{
- public:
-  // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize)
-  {
-    return static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->Evaluate(coordinates,
-        begin, batchSize);
-  }
-};
-
-/**
- * If we have a decomposable EvaluateWithGradient() but not a decomposable
- * Evaluate(), add a decomposable Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluate<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize)
-  {
-    arma::mat gradient; // This will be ignored.
-    return static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * The AddDecomposableEvaluateConst mixin class will add a decomposable const
- * Evaluate() method if a decomposable const EvaluateWithGradient() function
- * exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientConstForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateConstForm>::value>
-class AddDecomposableEvaluateConst
-{
- public:
-  // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&, const size_t, const size_t)
-      const;
-};
-
-/**
- * Reflect the existing Evaluate().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableEvaluateConst<FunctionType,
-    HasDecomposableEvaluateWithGradient, true>
-{
- public:
-  // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize) const
-  {
-    return static_cast<const FunctionType*>(
-        static_cast<const Function<FunctionType>*>(this))->Evaluate(coordinates,
-        begin, batchSize);
-  }
-};
-
-/**
- * If we have a decomposable const EvaluateWithGradient() but not a decomposable
- * const Evaluate(), add a decomposable const Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateConst<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize) const
-  {
-    arma::mat gradient; // This will be ignored.
-    return
-        static_cast<const Function<FunctionType>*>(this)->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * The AddDecomposableEvaluateStatic mixin class will add a decomposable static
- * Evaluate() method if a decomposable static EvaluateWithGradient() function
- * exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientStaticForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateStaticForm>::value>
-class AddDecomposableEvaluateStatic
-{
- public:
-  // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&, const size_t) const;
-};
-
-/**
- * Reflect the existing Evaluate().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableEvaluateStatic<FunctionType,
-    HasDecomposableEvaluateWithGradient, true>
-{
- public:
-  // Reflect the existing Evaluate().
-  static double Evaluate(const arma::mat& coordinates,
-                         const size_t begin,
-                         const size_t batchSize)
-  {
-    return FunctionType::Evaluate(coordinates, begin, batchSize);
-  }
-};
-
-/**
- * If we have a decomposable EvaluateWithGradient() but not a decomposable
- * Evaluate(), add a decomposable Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateStatic<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  static double Evaluate(const arma::mat& coordinates,
-                         const size_t begin,
-                         const size_t batchSize)
-  {
-    arma::mat gradient; // This will be ignored.
-    return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-} // namespace ens
-
-#endif

include/ensmallen_bits/function/add_decomposable_evaluate_const.hpp

@@ -1,165 +0,0 @@
-/**
- * @file add_decomposable_evaluate.hpp
- * @author Ryan Curtin
- *
- * Adds a decomposable Evaluate() function if a decomposable
- * EvaluateWithGradient() function exists.
- *
- * ensmallen is free software; you may redistribute it and/or modify it under
- * the terms of the 3-clause BSD license.  You should have received a copy of
- * the 3-clause BSD license along with ensmallen.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_HPP
-#define ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_HPP
-
-#include "traits.hpp"
-
-namespace ens {
-
-/**
- * The AddDecomposableEvaluate mixin class will add a decomposable Evaluate()
- * method if a decomposable EvaluateWithGradient() function exists, or nothing
- * otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateForm>::value>
-class AddDecomposableEvaluate
-{
- public:
-  // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&, const size_t);
-};
-
-/**
- * Reflect the existing Evaluate().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableEvaluate<FunctionType, HasDecomposableEvaluateWithGradient,
-    true>
-{
- public:
-  // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates, const size_t index)
-  {
-    return static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->Evaluate(coordinates,
-        index);
-  }
-};
-
-/**
- * If we have a decomposable EvaluateWithGradient() but not a decomposable
- * Evaluate(), add a decomposable Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluate<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize)
-  {
-    arma::mat gradient; // This will be ignored.
-    return static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * The AddDecomposableEvaluateConst mixin class will add a decomposable const
- * Evaluate() method if a decomposable const EvaluateWithGradient() function
- * exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientConstForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateConstForm>::value>
-class AddDecomposableEvaluateConst { };
-
-/**
- * If we have a decomposable const EvaluateWithGradient() but not a decomposable
- * const Evaluate(), add a decomposable const Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateConst<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize) const
-  {
-    arma::mat gradient; // This will be ignored.
-    return
-        static_cast<const Function<FunctionType>*>(this)->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * The AddDecomposableEvaluateStatic mixin class will add a decomposable static
- * Evaluate() method if a decomposable static EvaluateWithGradient() function
- * exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientStaticForm>::value,
-         bool HasDecomposableEvaluate =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateStaticForm>::value>
-class AddDecomposableEvaluateStatic { };
-
-/**
- * If we have a decomposable EvaluateWithGradient() but not a decomposable
- * Evaluate(), add a decomposable Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateStatic<FunctionType, true, false>
-{
- public:
-  /**
-   * Return the objective function for the given coordinates, starting at the
-   * given decomposable function using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of first function to evaluate.
-   * @param batchSize Number of functions to evaluate.
-   */
-  static double Evaluate(const arma::mat& coordinates,
-                         const size_t begin,
-                         const size_t batchSize)
-  {
-    arma::mat gradient; // This will be ignored.
-    return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-} // namespace ens
-
-#endif

include/ensmallen_bits/function/add_decomposable_evaluate_with_gradient.hpp

@@ -1,259 +0,0 @@
-/**
- * @file add_decomposable_evaluate_with_gradient.hpp
- * @author Ryan Curtin
- *
- * Adds a decomposable EvaluateWithGradient() function if both a decomposable
- * Evaluate() and a decomposable Gradient() function exist.
- *
- * ensmallen is free software; you may redistribute it and/or modify it under
- * the terms of the 3-clause BSD license.  You should have received a copy of
- * the 3-clause BSD license along with ensmallen.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_W_GRADIENT_HPP
-#define ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_W_GRADIENT_HPP
-
-#include "traits.hpp"
-
-namespace ens {
-
-/**
- * The AddDecomposableEvaluateWithGradient mixin class will add a decomposable
- * EvaluateWithGradient() method if a decomposable Evaluate() method and a
- * decomposable Gradient() method exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         // Check if there is at least one non-const Evaluate() or Gradient().
-         bool HasDecomposableEvaluateGradient = traits::HasNonConstSignatures<
-             FunctionType,
-             traits::HasEvaluate,
-             traits::DecomposableEvaluateForm,
-             traits::DecomposableEvaluateConstForm,
-             traits::DecomposableEvaluateStaticForm,
-             traits::HasGradient,
-             traits::DecomposableGradientForm,
-             traits::DecomposableGradientConstForm,
-             traits::DecomposableGradientStaticForm>::value,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientForm>::value>
-class AddDecomposableEvaluateWithGradient
-{
- public:
-  // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
-  // object.
-  double EvaluateWithGradient(traits::UnconstructableType&, const size_t,
-      const size_t);
-};
-
-/**
- * Reflect the existing EvaluateWithGradient().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateGradient>
-class AddDecomposableEvaluateWithGradient<FunctionType,
-    HasDecomposableEvaluateGradient, true>
-{
- public:
-  // Reflect the existing Evaluate().
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize)
-  {
-    return static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * If we have a both decomposable Evaluate() and a decomposable Gradient() but
- * not a decomposable EvaluateWithGradient(), add a decomposable
- * EvaluateWithGradient() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradient<FunctionType, true, false>
-{
- public:
-  /**
-   * Return both the evaluated objective function and its gradient, storing the
-   * gradient in the given matrix, starting at the given decomposable function
-   * and using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of decomposable function to begin with.
-   * @param gradient Matrix to store the gradient into.
-   * @param batchSize Number of decomposable functions to evaluate.
-   */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize)
-  {
-    const double objective =
-        static_cast<Function<FunctionType>*>(this)->Evaluate(coordinates, begin,
-        batchSize);
-    static_cast<Function<FunctionType>*>(this)->Gradient(coordinates, begin,
-        gradient, batchSize);
-    return objective;
-  }
-};
-
-/**
- * The AddDecomposableEvaluateWithGradientConst mixin class will add a
- * decomposable const EvaluateWithGradient() method if both a decomposable const
- * Evaluate() and a decomposable const Gradient() function exist, or nothing
- * otherwise.
- */
-template<typename FunctionType,
-         // Check if there is at least one const Evaluate() or Gradient().
-         bool HasDecomposableEvaluateGradient = traits::HasConstSignatures<
-             FunctionType,
-             traits::HasEvaluate,
-             traits::DecomposableEvaluateConstForm,
-             traits::DecomposableEvaluateStaticForm,
-             traits::HasGradient,
-             traits::DecomposableGradientConstForm,
-             traits::DecomposableGradientStaticForm>::value,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientConstForm>::value>
-class AddDecomposableEvaluateWithGradientConst
-{
- public:
-  // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
-  // object.
-  double EvaluateWithGradient(traits::UnconstructableType&, const size_t,
-      const size_t) const;
-};
-
-/**
- * Reflect the existing EvaluateWithGradient().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateGradient>
-class AddDecomposableEvaluateWithGradientConst<FunctionType,
-    HasDecomposableEvaluateGradient, true>
-{
- public:
-  // Reflect the existing Evaluate().
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize) const
-  {
-    return static_cast<const FunctionType*>(
-        static_cast<const Function<FunctionType>*>(this))->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * If we have both a decomposable const Evaluate() and a decomposable const
- * Gradient() but not a decomposable const EvaluateWithGradient(), add a
- * decomposable const EvaluateWithGradient() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradientConst<FunctionType, true, false>
-{
- public:
-  /**
-   * Return both the evaluated objective function and its gradient, storing the
-   * gradient in the given matrix, starting at the given decomposable function
-   * and using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of decomposable function to begin with.
-   * @param gradient Matrix to store the gradient into.
-   * @param batchSize Number of decomposable functions to evaluate.
-   */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize) const
-  {
-    const double objective =
-        static_cast<const Function<FunctionType>*>(this)->Evaluate(coordinates,
-        begin, batchSize);
-    static_cast<const Function<FunctionType>*>(this)->Gradient(coordinates,
-        begin, gradient, batchSize);
-    return objective;
-  }
-};
-
-/**
- * The AddDecomposableEvaluateWithGradientStatic mixin class will add a
- * decomposable static EvaluateWithGradient() method if both a decomposable
- * static Evaluate() and a decomposable static gradient() function exist, or
- * nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateGradient =
-             traits::HasEvaluate<FunctionType,
-                 traits::DecomposableEvaluateStaticForm>::value &&
-             traits::HasGradient<FunctionType,
-                 traits::DecomposableGradientStaticForm>::value,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientStaticForm>::value>
-class AddDecomposableEvaluateWithGradientStatic
-{
- public:
-  // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
-  // object.
-  static double EvaluateWithGradient(traits::UnconstructableType&, const size_t,
-      const size_t);
-};
-
-/**
- * Reflect the existing EvaluateWithGradient().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateGradient>
-class AddDecomposableEvaluateWithGradientStatic<FunctionType,
-    HasDecomposableEvaluateGradient, true>
-{
- public:
-  // Reflect the existing Evaluate().
-  static double EvaluateWithGradient(const arma::mat& coordinates,
-                                     const size_t begin,
-                                     arma::mat& gradient,
-                                     const size_t batchSize)
-  {
-    return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-/**
- * If we have a decomposable static Evaluate() and a decomposable static
- * Gradient() but not a decomposable static EvaluateWithGradient(), add a
- * decomposable static Gradient() method.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradientStatic<FunctionType, true, false>
-{
- public:
-  /**
-   * Return both the evaluated objective function and its gradient, storing the
-   * gradient in the given matrix, starting at the given decomposable function
-   * and using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of decomposable function to begin with.
-   * @param gradient Matrix to store the gradient into.
-   * @param batchSize Number of decomposable functions to evaluate.
-   */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize) const
-  {
-    const double objective = FunctionType::Evaluate(coordinates, begin,
-        batchSize);
-    FunctionType::Gradient(coordinates, begin, gradient, batchSize);
-    return objective;
-  }
-};
-
-} // namespace ens
-
-#endif

include/ensmallen_bits/function/add_decomposable_evaluate_with_gradient_const.hpp

@@ -1,185 +0,0 @@
-/**
- * @file add_decomposable_evaluate_with_gradient.hpp
- * @author Ryan Curtin
- *
- * Add decomposable variants of Evaluate(), Gradient(), and
- * EvaluateWithGradient().
- *
- * ensmallen is free software; you may redistribute it and/or modify it under
- * the terms of the 3-clause BSD license.  You should have received a copy of
- * the 3-clause BSD license along with ensmallen.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef ENSMALLEN_ADD_DECOMPOSABLE_EVALUATE_WITH_GRADIENT_CONST_HPP
-#define ENSMALLEN_ADD_DECOMPOSABLE_EVALUATE_WITH_GRADIENT_CONST_HPP
-
-#include "traits.hpp"
-
-namespace ens {
-
-namespace aux {
-
-template<typename FunctionType>
-using DecomposableEvaluateForm = double(FunctionType::*)(const arma::mat&,
-                                                         const size_t,
-                                                         const size_t);
-
-template<typename FunctionType>
-using DecomposableEvaluateConstForm =
-    double(FunctionType::*)(const arma::mat&, const size_t, const size_t) const;
-
-template<typename FunctionType>
-using DecomposableEvaluateStaticForm = double(*)(const arma::mat&,
-                                                 size_t,
-                                                 size_t);
-
-template<typename FunctionType>
-using DecomposableGradientForm = void(FunctionType::*)(const arma::mat&,
-                                                       const size_t,
-                                                       arma::mat&,
-                                                       const size_t);
-
-template<typename FunctionType>
-using DecomposableGradientConstForm =
-    void(FunctionType::*)(const arma::mat&,
-                          const size_t,
-                          arma::mat&,
-                          const size_t) const;
-
-template<typename FunctionType, typename... Ts>
-using DecomposableGradientStaticForm = void(*)(const arma::mat&,
-                                               const size_t,
-                                               arma::mat&);
-
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientForm =
-    double(FunctionType::*)(const arma::mat&,
-                            const size_t,
-                            arma::mat&,
-                            const size_t);
-
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientConstForm =
-    void(FunctionType::*)(const arma::mat&,
-                          const size_t,
-                          arma::mat&,
-                          const size_t) const;
-
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientStaticForm =
-    double(*)(const arma::mat&, const size_t, arma::mat&, const size_t);
-
-} // namespace aux
-
-/**
- * The AddDecomposableEvaluateWithGradient mixin class will provide a
- * decomposable Evaluate() and Gradient() method if the given class has a
- * decomposable EvaluateWithGradient() method, or it will provide a decomposable
- * EvaluateWithGradient() method if the class has a decomposable Evaluate() and
- * Gradient() method, or it will provide nothing in any other case.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluate =
-             aux::HasEvaluate<FunctionType,
-                 aux::DecomposableEvaluateForm>::value ||
-             aux::HasEvaluate<FunctionType,
-                 aux::DecomposableEvaluateConstForm>::value ||
-             aux::HasEvaluate<FunctionType,
-                 aux::DecomposableEvaluateStaticForm>::value,
-         bool HasGradient =
-             aux::HasGradient<FunctionType,
-                 aux::DecomposableGradientForm>::value ||
-             aux::HasGradient<FunctionType,
-                 aux::DecomposableGradientConstForm>::value ||
-             aux::HasGradient<FunctionType,
-                 aux::DecomposableGradientStaticForm>::value,
-         bool HasEvaluateWithGradient =
-             aux::HasEvaluateWithGradient<FunctionType,
-                 aux::DecomposableEvaluateWithGradientForm>::value ||
-             aux::HasEvaluateWithGradient<FunctionType,
-                 aux::DecomposableEvaluateWithGradientConstForm>::value ||
-             aux::HasEvaluateWithGradient<FunctionType,
-                 aux::DecomposableEvaluateWithGradientStaticForm>::value>
-class AddDecomposableEvaluateWithGradient : public FunctionType { };
-
-/**
- * If the FunctionType has Evaluate() and Gradient() but not
- * EvaluateWithGradient(), we will provide the latter.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradient<FunctionType, true, true, false> :
-    public FunctionType
-{
- public:
-  /**
-   * Return both the evaluated objective function and its gradient, storing the
-   * gradient in the given matrix.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param gradient Matrix to store the gradient into.
-   */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              const size_t begin,
-                              arma::mat& gradient,
-                              const size_t batchSize)
-  {
-    const double objective = FunctionType::Evaluate(coordinates, begin,
-        batchSize);
-    FunctionType::Gradient(coordinates, begin, gradient, batchSize);
-    return objective;
-  }
-};
-
-/**
- * If the FunctionType has EvaluateWithGradient() but not Evaluate(), provide
- * that function.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradient<FunctionType, false, true, true> :
-    public FunctionType
-{
- public:
-  /**
-   * Return the objective function for the given coordinates.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   */
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t begin,
-                  const size_t batchSize)
-  {
-    arma::mat gradient; // This will be ignored.
-    return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-/**
- * If the FunctionType has EvaluateWithGradient() but not Gradient(), provide
- * that function.
- */
-template<typename FunctionType>
-class AddDecomposableEvaluateWithGradient<FunctionType, true, false, true> :
-      public FunctionType
-{
- public:
-  /**
-   * Calculate the gradient and store it in the given matrix.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param gradient Matrix to store the gradient into.
-   */
-  void Gradient(const arma::mat& coordinates,
-                const size_t begin,
-                arma::mat& gradient,
-                const size_t batchSize)
-  {
-    // The returned objective value will be ignored.
-    (void) FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-} // namespace ens
-
-#endif

include/ensmallen_bits/function/add_decomposable_gradient.hpp

@@ -1,224 +0,0 @@
-/**
- * @file add_decomposable_gradient.hpp
- * @author Ryan Curtin
- *
- * Adds a decomposable Gradient() function if a decomposable
- * EvaluateWithGradient() function exists.
- *
- * ensmallen is free software; you may redistribute it and/or modify it under
- * the terms of the 3-clause BSD license.  You should have received a copy of
- * the 3-clause BSD license along with ensmallen.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_GRADIENT_HPP
-#define ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_GRADIENT_HPP
-
-#include "traits.hpp"
-
-namespace ens {
-
-/**
- * The AddDecomposableGradient mixin class will add a decomposable Gradient()
- * method if a decomposable EvaluateWithGradient() function exists, or nothing
- * otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientForm>::value,
-         bool HasDecomposableGradient =
-             traits::HasGradient<FunctionType,
-                 traits::DecomposableGradientForm>::value>
-class AddDecomposableGradient
-{
- public:
-  // Provide a dummy overload so the name 'Gradient' exists for this object.
-  void Gradient(traits::UnconstructableType&, const size_t, const size_t);
-};
-
-/**
- * Reflect the existing Gradient().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableGradient<FunctionType, HasDecomposableEvaluateWithGradient,
-    true>
-{
- public:
-  // Reflect the existing Gradient().
-  void Gradient(const arma::mat& coordinates,
-                const size_t begin,
-                arma::mat& gradient,
-                const size_t batchSize)
-  {
-    static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->Gradient(coordinates,
-        begin, gradient, batchSize);
-  }
-};
-
-/**
- * If we have a decomposable EvaluateWithGradient() but not a decomposable
- * Gradient(), add a decomposable Evaluate() method.
- */
-template<typename FunctionType>
-class AddDecomposableGradient<FunctionType, true, false>
-{
- public:
-  /**
-   * Calculate the gradient and store it in the given matrix, starting at the
-   * given decomposable function index and using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of decomposable function to start at.
-   * @param gradient Matrix to store the gradient into.
-   * @param batchSize Number of decomposable functions to calculate for.
-   */
-  void Gradient(const arma::mat& coordinates,
-                const size_t begin,
-                arma::mat& gradient,
-                const size_t batchSize)
-  {
-    // The returned objective value will be ignored.
-    (void) static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
-        coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * The AddDecomposableGradientConst mixin class will add a decomposable const
- * Gradient() method if a decomposable const EvaluateWithGradient() function
- * exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientConstForm>::value,
-         bool HasDecomposableGradient =
-             traits::HasGradient<FunctionType,
-                 traits::DecomposableGradientConstForm>::value>
-class AddDecomposableGradientConst
-{
- public:
-  // Provide a dummy overload so the name 'Gradient' exists for this object.
-  void Gradient(traits::UnconstructableType&, const size_t, const size_t) const;
-};
-
-/**
- * Reflect the existing Gradient().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableGradientConst<FunctionType,
-    HasDecomposableEvaluateWithGradient, true>
-{
- public:
-  // Reflect the existing Gradient().
-  void Gradient(const arma::mat& coordinates,
-                const size_t begin,
-                arma::mat& gradient,
-                const size_t batchSize) const
-  {
-    static_cast<const FunctionType*>(
-        static_cast<const Function<FunctionType>*>(this))->Gradient(coordinates,
-        begin, gradient, batchSize);
-  }
-};
-
-/**
- * If we have a decomposable const EvaluateWithGradient() but not a decomposable
- * const Gradient(), add a decomposable const Gradient() method.
- */
-template<typename FunctionType>
-class AddDecomposableGradientConst<FunctionType, true, false>
-{
- public:
-  /**
-   * Calculate the gradient and store it in the given matrix, starting at the
-   * given decomposable function index and using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of decomposable function to start at.
-   * @param gradient Matrix to store the gradient into.
-   * @param batchSize Number of decomposable functions to calculate for.
-   */
-  void Gradient(const arma::mat& coordinates,
-                const size_t begin,
-                arma::mat& gradient,
-                const size_t batchSize) const
-  {
-    // The returned objective value will be ignored.
-    (void) static_cast<
-        const Function<FunctionType>*>(this)->EvaluateWithGradient(coordinates,
-        begin, gradient, batchSize);
-  }
-};
-
-/**
- * The AddDecomposableEvaluateStatic mixin class will add a decomposable static
- * Gradient() method if a decomposable static EvaluateWithGradient() function
- * exists, or nothing otherwise.
- */
-template<typename FunctionType,
-         bool HasDecomposableEvaluateWithGradient =
-             traits::HasEvaluateWithGradient<FunctionType,
-                 traits::DecomposableEvaluateWithGradientStaticForm>::value,
-         bool HasDecomposableGradient =
-             traits::HasGradient<FunctionType,
-                 traits::DecomposableGradientStaticForm>::value>
-class AddDecomposableGradientStatic
-{
- public:
-  // Provide a dummy overload so the name 'Gradient' exists for this object.
-  static void Gradient(traits::UnconstructableType&,
-                       const size_t,
-                       const size_t);
-};
-
-/**
- * Reflect the existing Gradient().
- */
-template<typename FunctionType, bool HasDecomposableEvaluateWithGradient>
-class AddDecomposableGradientStatic<FunctionType,
-    HasDecomposableEvaluateWithGradient, true>
-{
- public:
-  // Reflect the existing Gradient().
-  static void Gradient(const arma::mat& coordinates,
-                       const size_t begin,
-                       arma::mat& gradient,
-                       const size_t batchSize)
-  {
-    FunctionType::Gradient(coordinates, begin, gradient, batchSize);
-  }
-};
-
-/**
- * If we have a decomposable EvaluateWithGradient() but not a decomposable
- * Gradient(), add a decomposable Gradient() method.
- */
-template<typename FunctionType>
-class AddDecomposableGradientStatic<FunctionType, true, false>
-{
- public:
-  /**
-   * Calculate the gradient and store it in the given matrix, starting at the
-   * given decomposable function index and using the given batch size.
-   *
-   * @param coordinates Coordinates to evaluate the function at.
-   * @param begin Index of decomposable function to start at.
-   * @param gradient Matrix to store the gradient into.
-   * @param batchSize Number of decomposable functions to calculate for.
-   */
-  static void Gradient(const arma::mat& coordinates,
-                       const size_t begin,
-                       arma::mat& gradient,
-                       const size_t batchSize)
-  {
-    // The returned objective value will be ignored.
-    (void) FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
-        batchSize);
-  }
-};
-
-} // namespace ens
-
-#endif

include/ensmallen_bits/function/add_evaluate.hpp

@@ -22,30 +22,44 @@ namespace ens {
  * FunctionType has EvaluateWithGradient(), or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     EvaluateWithGradientForm
+              >::value,
          bool HasEvaluate =
-             traits::HasEvaluate<FunctionType, traits::EvaluateForm>::value>
+             traits::HasEvaluate<FunctionType,
+                  traits::TypedForms<MatType, GradType>::template
+                     EvaluateForm>::value>
 class AddEvaluate
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&);
+  typename MatType::elem_type Evaluate(traits::UnconstructableType&);
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddEvaluate<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddEvaluate<FunctionType,
+                  MatType,
+                  GradType,
+                  HasEvaluateWithGradient,
+                  true>
 {
  public:
   // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates)
+  typename MatType::elem_type Evaluate(const MatType& coordinates)
   {
-    return static_cast<FunctionType*>(static_cast<Function<FunctionType>*>(
-        this))->Evaluate(coordinates);
+    return static_cast<FunctionType*>(
+        static_cast<Function<FunctionType,
+                             MatType, GradType>*>(this))->Evaluate(coordinates);
   }
 };
 
@@ -53,8 +67,8 @@ class AddEvaluate<FunctionType, HasEvaluateWithGradient, true>
  * If we have EvaluateWithGradient() but no existing Evaluate(), add an
  * Evaluate() method.
  */
-template<typename FunctionType>
-class AddEvaluate<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluate<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -62,10 +76,12 @@ class AddEvaluate<FunctionType, true, false>
    *
    * @param coordinates Coordinates to evaluate the function at.
    */
-  double Evaluate(const arma::mat& coordinates)
+  typename MatType::elem_type Evaluate(const MatType& coordinates)
   {
-    arma::mat gradient; // This will be ignored.
-    return static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
+    GradType gradient; // This will be ignored.
+    return static_cast<Function<FunctionType,
+                                MatType,
+                                GradType>*>(this)->EvaluateWithGradient(
         coordinates, gradient);
   }
 };
@@ -76,32 +92,47 @@ class AddEvaluate<FunctionType, true, false>
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientConstForm>::value,
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                     EvaluateWithGradientConstForm
+             >::value,
          bool HasEvaluate =
              traits::HasEvaluate<FunctionType,
-                 traits::EvaluateConstForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                     EvaluateConstForm
+             >::value>
 class AddEvaluateConst
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  double Evaluate(traits::UnconstructableType&) const;
+  typename MatType::elem_type Evaluate(traits::UnconstructableType&) const;
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddEvaluateConst<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddEvaluateConst<FunctionType,
+                       MatType,
+                       GradType,
+                       HasEvaluateWithGradient,
+                       true>
 {
  public:
   // Reflect the existing Evaluate().
-  double Evaluate(const arma::mat& coordinates) const
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const
   {
-    return static_cast<const FunctionType*>(static_cast<const
-Function<FunctionType>*>(this))->Evaluate(
-        coordinates);
+    return static_cast<const FunctionType*>(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->Evaluate(coordinates);
   }
 };
 
@@ -109,8 +140,8 @@ Function<FunctionType>*>(this))->Evaluate(
  * If we have EvaluateWithGradient() but no existing Evaluate(), add an
  * Evaluate() without a using directive to make the base Evaluate() accessible.
  */
-template<typename FunctionType>
-class AddEvaluateConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateConst<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -118,12 +149,14 @@ class AddEvaluateConst<FunctionType, true, false>
    *
    * @param coordinates Coordinates to evaluate the function at.
    */
-  double Evaluate(const arma::mat& coordinates) const
+  typename MatType::elem_type Evaluate(const MatType& coordinates) const
   {
-    arma::mat gradient; // This will be ignored.
+    GradType gradient; // This will be ignored.
     return static_cast<
-        const Function<FunctionType>*>(this)->EvaluateWithGradient(coordinates,
-        gradient);
+        const Function<FunctionType,
+                       MatType,
+                       GradType>*>(this)->EvaluateWithGradient(coordinates,
+                                                               gradient);
   }
 };
 
@@ -133,28 +166,43 @@ class AddEvaluateConst<FunctionType, true, false>
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientStaticForm>::value,
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                 EvaluateWithGradientStaticForm
+             >::value,
          bool HasEvaluate =
              traits::HasEvaluate<FunctionType,
-                 traits::EvaluateStaticForm>::value>
+                 traits::TypedForms<MatType, GradType>::template
+                 EvaluateStaticForm
+             >::value>
 class AddEvaluateStatic
 {
  public:
   // Provide a dummy overload so the name 'Evaluate' exists for this object.
-  static double Evaluate(traits::UnconstructableType&);
+  static typename MatType::elem_type Evaluate(traits::UnconstructableType&);
 };
 
 /**
  * Reflect the existing Evaluate().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddEvaluateStatic<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddEvaluateStatic<FunctionType,
+                        MatType,
+                        GradType,
+                        HasEvaluateWithGradient,
+                        true>
 {
  public:
   // Reflect the existing Evaluate().
-  static double Evaluate(const arma::mat& coordinates)
+  static typename MatType::elem_type Evaluate(
+      const MatType& coordinates)
   {
     return FunctionType::Evaluate(coordinates);
   }
@@ -164,8 +212,8 @@ class AddEvaluateStatic<FunctionType, HasEvaluateWithGradient, true>
  * If we have EvaluateWithGradient() but no existing Evaluate(), add an
  * Evaluate() without a using directive to make the base Evaluate() accessible.
  */
-template<typename FunctionType>
-class AddEvaluateStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateStatic<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -173,9 +221,9 @@ class AddEvaluateStatic<FunctionType, true, false>
    *
    * @param coordinates Coordinates to evaluate the function at.
    */
-  static double Evaluate(const arma::mat& coordinates)
+  static typename MatType::elem_type Evaluate(const MatType& coordinates)
   {
-    arma::mat gradient; // This will be ignored.
+    GradType gradient; // This will be ignored.
     return FunctionType::EvaluateWithGradient(coordinates, gradient);
   }
 };

include/ensmallen_bits/function/add_evaluate_with_gradient.hpp

@@ -24,40 +24,55 @@ namespace ens {
  * and Gradient(), or it will provide nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          // Check if there is at least one non-const Evaluate() or Gradient().
          bool HasEvaluateGradient = traits::HasNonConstSignatures<
              FunctionType,
              traits::HasEvaluate,
-             traits::EvaluateForm,
-             traits::EvaluateConstForm,
-             traits::EvaluateStaticForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateConstForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateStaticForm,
              traits::HasGradient,
-             traits::GradientForm,
-             traits::GradientConstForm,
-             traits::GradientStaticForm>::value,
+             traits::TypedForms<MatType, GradType>::template GradientForm,
+             traits::TypedForms<MatType, GradType>::template GradientConstForm,
+             traits::TypedForms<MatType, GradType>::template GradientStaticForm
+         >::value,
          bool HasEvaluateWithGradient = traits::HasEvaluateWithGradient<
              FunctionType,
-             traits::EvaluateWithGradientForm>::value>
+             traits::TypedForms<MatType, GradType>::template
+                 EvaluateWithGradientForm>::value>
 class AddEvaluateWithGradient
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  double EvaluateWithGradient(traits::UnconstructableType&);
+  typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&);
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasEvaluateGradient>
-class AddEvaluateWithGradient<FunctionType, HasEvaluateGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateGradient>
+class AddEvaluateWithGradient<FunctionType,
+                              MatType,
+                              GradType,
+                              HasEvaluateGradient,
+                              true>
 {
  public:
   // Reflect the existing EvaluateWithGradient().
-  double EvaluateWithGradient(const arma::mat& coordinates, arma::mat& gradient)
+  typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates, GradType& gradient)
   {
     return static_cast<FunctionType*>(
-        static_cast<Function<FunctionType>*>(this))->EvaluateWithGradient(
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->EvaluateWithGradient(
         coordinates, gradient);
   }
 };
@@ -66,8 +81,8 @@ class AddEvaluateWithGradient<FunctionType, HasEvaluateGradient, true>
  * If the FunctionType has Evaluate() and Gradient(), provide
  * EvaluateWithGradient().
  */
-template<typename FunctionType>
-class AddEvaluateWithGradient<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateWithGradient<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -77,12 +92,15 @@ class AddEvaluateWithGradient<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              arma::mat& gradient)
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   GradType& gradient)
   {
-    const double objective =
-        static_cast<Function<FunctionType>*>(this)->Evaluate(coordinates);
-    static_cast<Function<FunctionType>*>(this)->Gradient(coordinates, gradient);
+    const typename MatType::elem_type objective =
+        static_cast<Function<FunctionType,
+                             MatType, GradType>*>(this)->Evaluate(coordinates);
+    static_cast<Function<FunctionType,
+                         MatType,
+                         GradType>*>(this)->Gradient(coordinates, gradient);
     return objective;
   }
 };
@@ -93,39 +111,54 @@ class AddEvaluateWithGradient<FunctionType, true, false>
  * Evaluate() const and Gradient() const, or it will provide nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          // Check if there is at least one const Evaluate() or Gradient().
          bool HasEvaluateGradient = traits::HasConstSignatures<
              FunctionType,
              traits::HasEvaluate,
-             traits::EvaluateConstForm,
-             traits::EvaluateStaticForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateConstForm,
+             traits::TypedForms<MatType, GradType>::template EvaluateStaticForm,
              traits::HasGradient,
-             traits::GradientConstForm,
-             traits::GradientStaticForm>::value,
+             traits::TypedForms<MatType, GradType>::template GradientConstForm,
+             traits::TypedForms<MatType, GradType>::template GradientStaticForm
+         >::value,
          bool HasEvaluateWithGradient = traits::HasEvaluateWithGradient<
              FunctionType,
-             traits::EvaluateWithGradientConstForm>::value>
+             traits::TypedForms<
+                 MatType, GradType
+             >::template EvaluateWithGradientConstForm>::value>
 class AddEvaluateWithGradientConst
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  double EvaluateWithGradient(traits::UnconstructableType&) const;
+  typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&) const;
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasEvaluateGradient>
-class AddEvaluateWithGradientConst<FunctionType, HasEvaluateGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateGradient>
+class AddEvaluateWithGradientConst<FunctionType,
+                                   MatType,
+                                   GradType,
+                                   HasEvaluateGradient,
+                                   true>
 {
  public:
   // Reflect the existing EvaluateWithGradient().
-  double EvaluateWithGradient(const arma::mat& coordinates, arma::mat& gradient)
-      const
+  typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates, GradType& gradient) const
   {
     return static_cast<const FunctionType*>(
-        static_cast<const Function<FunctionType>*>(this))->EvaluateWithGradient(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->EvaluateWithGradient(
         coordinates, gradient);
   }
 };
@@ -134,8 +167,8 @@ class AddEvaluateWithGradientConst<FunctionType, HasEvaluateGradient, true>
  * If the FunctionType has Evaluate() const and Gradient() const, provide
  * EvaluateWithGradient() const.
  */
-template<typename FunctionType>
-class AddEvaluateWithGradientConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateWithGradientConst<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -145,13 +178,17 @@ class AddEvaluateWithGradientConst<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  double EvaluateWithGradient(const arma::mat& coordinates,
-                              arma::mat& gradient) const
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   GradType& gradient) const
   {
-    const double objective =
-        static_cast<const Function<FunctionType>*>(this)->Evaluate(coordinates);
-    static_cast<const Function<FunctionType>*>(this)->Gradient(coordinates,
-        gradient);
+    const typename MatType::elem_type objective =
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this)->Evaluate(coordinates);
+    static_cast<const Function<FunctionType,
+                               MatType,
+                               GradType>*>(this)->Gradient(coordinates,
+                                                           gradient);
     return objective;
   }
 };
@@ -163,32 +200,49 @@ class AddEvaluateWithGradientConst<FunctionType, true, false>
  * otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateGradient =
              traits::HasEvaluate<FunctionType,
-                 traits::EvaluateStaticForm>::value &&
+                 traits::TypedForms<MatType, GradType>::template
+                     EvaluateStaticForm
+             >::value &&
              traits::HasGradient<FunctionType,
-                 traits::GradientStaticForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     GradientStaticForm
+             >::value,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientStaticForm>::value>
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                     EvaluateWithGradientStaticForm
+             >::value>
 class AddEvaluateWithGradientStatic
 {
  public:
   // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
   // object.
-  static double EvaluateWithGradient(traits::UnconstructableType&);
+  static typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&);
 };
 
 /**
  * Reflect the existing EvaluateWithGradient().
  */
-template<typename FunctionType, bool HasEvaluateGradient>
-class AddEvaluateWithGradientStatic<FunctionType, HasEvaluateGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateGradient>
+class AddEvaluateWithGradientStatic<FunctionType,
+                                    MatType,
+                                    GradType,
+                                    HasEvaluateGradient,
+                                    true>
 {
  public:
   // Reflect the existing EvaluateWithGradient().
-  static double EvaluateWithGradient(const arma::mat& coordinates,
-                                     arma::mat& gradient)
+  static typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates, GradType& gradient)
   {
     return FunctionType::EvaluateWithGradient(coordinates, gradient);
   }
@@ -198,8 +252,12 @@ class AddEvaluateWithGradientStatic<FunctionType, HasEvaluateGradient, true>
  * If the FunctionType has static Evaluate() and static Gradient(), provide
  * static EvaluateWithGradient().
  */
-template<typename FunctionType>
-class AddEvaluateWithGradientStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddEvaluateWithGradientStatic<FunctionType,
+                                    MatType,
+                                    GradType,
+                                    true,
+                                    false>
 {
  public:
   /**
@@ -209,10 +267,11 @@ class AddEvaluateWithGradientStatic<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  static double EvaluateWithGradient(const arma::mat& coordinates,
-                                     arma::mat& gradient)
+  static typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates, GradType& gradient)
   {
-    const double objective = FunctionType::Evaluate(coordinates);
+    const typename MatType::elem_type objective =
+        FunctionType::Evaluate(coordinates);
     FunctionType::Gradient(coordinates, gradient);
     return objective;
   }

include/ensmallen_bits/function/add_gradient.hpp

@@ -22,11 +22,16 @@ namespace ens {
  * FunctionType has EvaluateWithGradient(), or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientForm>::value,
+                 traits::TypedForms<MatType, GradType>::template
+                     EvaluateWithGradientForm
+             >::value,
          bool HasGradient = traits::HasGradient<FunctionType,
-             traits::GradientForm>::value>
+             traits::TypedForms<MatType, GradType>::template 
+                     GradientForm>::value>
 class AddGradient
 {
  public:
@@ -37,15 +42,25 @@ class AddGradient
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddGradient<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddGradient<FunctionType,
+                  MatType,
+                  GradType,
+                  HasEvaluateWithGradient,
+                  true>
 {
  public:
   // Reflect the existing Gradient().
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient)
+  void Gradient(const MatType& coordinates, GradType& gradient)
   {
-    static_cast<FunctionType*>(static_cast<Function<FunctionType>*>(
-        this))->Gradient(coordinates, gradient);
+    static_cast<FunctionType*>(
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->Gradient(coordinates,
+                                                          gradient);
   }
 };
 
@@ -53,8 +68,8 @@ class AddGradient<FunctionType, HasEvaluateWithGradient, true>
  * If we have EvaluateWithGradient() but no existing Gradient(), add an
  * Gradient() without a using directive to make the base Gradient() accessible.
  */
-template<typename FunctionType>
-class AddGradient<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddGradient<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -63,10 +78,12 @@ class AddGradient<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient)
+  void Gradient(const MatType& coordinates, GradType& gradient)
   {
     // The returned objective value will be ignored.
-    (void) static_cast<Function<FunctionType>*>(this)->EvaluateWithGradient(
+    (void) static_cast<Function<FunctionType,
+                                MatType,
+                                GradType>*>(this)->EvaluateWithGradient(
         coordinates, gradient);
   }
 };
@@ -76,11 +93,17 @@ class AddGradient<FunctionType, true, false>
  * given FunctionType has EvaluateWithGradient() const, or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientConstForm>::value,
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                     EvaluateWithGradientConstForm
+             >::value,
          bool HasGradient = traits::HasGradient<FunctionType,
-             traits::GradientConstForm>::value>
+             traits::TypedForms<MatType, GradType>::template GradientConstForm
+         >::value>
 class AddGradientConst
 {
  public:
@@ -91,16 +114,25 @@ class AddGradientConst
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddGradientConst<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddGradientConst<FunctionType,
+                       MatType,
+                       GradType,
+                       HasEvaluateWithGradient,
+                       true>
 {
  public:
   // Reflect the existing Gradient().
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const
+  void Gradient(const MatType& coordinates, GradType& gradient) const
   {
-    static_cast<const FunctionType*>(static_cast<const
-Function<FunctionType>*>(this))->Gradient(coordinates,
-        gradient);
+    static_cast<const FunctionType*>(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->Gradient(coordinates,
+                                                                gradient);
   }
 };
 
@@ -108,8 +140,8 @@ Function<FunctionType>*>(this))->Gradient(coordinates,
  * If we have EvaluateWithGradient() but no existing Gradient(), add a
  * Gradient() without a using directive to make the base Gradient() accessible.
  */
-template<typename FunctionType>
-class AddGradientConst<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddGradientConst<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -118,12 +150,14 @@ class AddGradientConst<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const
+  void Gradient(const MatType& coordinates, GradType& gradient) const
   {
     // The returned objective value will be ignored.
     (void) static_cast<
-        const Function<FunctionType>*>(this)->EvaluateWithGradient(coordinates,
-        gradient);
+        const Function<FunctionType,
+                       MatType,
+                       GradType>*>(this)->EvaluateWithGradient(coordinates,
+                                                               gradient);
   }
 };
 
@@ -132,11 +166,17 @@ class AddGradientConst<FunctionType, true, false>
  * given FunctionType has static EvaluateWithGradient(), or nothing otherwise.
  */
 template<typename FunctionType,
+         typename MatType,
+         typename GradType,
          bool HasEvaluateWithGradient =
              traits::HasEvaluateWithGradient<FunctionType,
-                 traits::EvaluateWithGradientStaticForm>::value,
+                 traits::TypedForms<MatType,
+                                    GradType>::template
+                     EvaluateWithGradientStaticForm
+             >::value,
          bool HasGradient = traits::HasGradient<FunctionType,
-             traits::GradientStaticForm>::value>
+             traits::TypedForms<MatType, GradType>::template GradientStaticForm
+         >::value>
 class AddGradientStatic
 {
  public:
@@ -147,12 +187,19 @@ class AddGradientStatic
 /**
  * Reflect the existing Gradient().
  */
-template<typename FunctionType, bool HasEvaluateWithGradient>
-class AddGradientStatic<FunctionType, HasEvaluateWithGradient, true>
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasEvaluateWithGradient>
+class AddGradientStatic<FunctionType,
+                        MatType,
+                        GradType,
+                        HasEvaluateWithGradient,
+                        true>
 {
  public:
   // Reflect the existing Gradient().
-  static void Gradient(const arma::mat& coordinates, arma::mat& gradient)
+  static void Gradient(const MatType& coordinates, GradType& gradient)
   {
     FunctionType::Gradient(coordinates, gradient);
   }
@@ -162,8 +209,8 @@ class AddGradientStatic<FunctionType, HasEvaluateWithGradient, true>
  * If we have EvaluateWithGradient() but no existing Gradient(), add a
  * Gradient() without a using directive to make the base Gradient() accessible.
  */
-template<typename FunctionType>
-class AddGradientStatic<FunctionType, true, false>
+template<typename FunctionType, typename MatType, typename GradType>
+class AddGradientStatic<FunctionType, MatType, GradType, true, false>
 {
  public:
   /**
@@ -172,7 +219,7 @@ class AddGradientStatic<FunctionType, true, false>
    * @param coordinates Coordinates to evaluate the function at.
    * @param gradient Matrix to store the gradient into.
    */
-  static void Gradient(const arma::mat& coordinates, arma::mat& gradient)
+  static void Gradient(const MatType& coordinates, GradType& gradient)
   {
     // The returned objective value will be ignored.
     (void) FunctionType::EvaluateWithGradient(coordinates, gradient);

include/ensmallen_bits/function/add_separable_evaluate.hpp

@@ -0,0 +1,251 @@
+/**
+ * @file add_separable_evaluate.hpp
+ * @author Ryan Curtin
+ *
+ * Adds a separable Evaluate() function if a separable
+ * EvaluateWithGradient() function exists.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_HPP
+#define ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_HPP
+
+#include "traits.hpp"
+
+namespace ens {
+
+/**
+ * The AddSeparableEvaluate mixin class will add a separable Evaluate()
+ * method if a separable EvaluateWithGradient() function exists, or nothing
+ * otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientForm
+             >::value,
+         bool HasSeparableEvaluate =
+             traits::HasEvaluate<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                      SeparableEvaluateForm>::value>
+class AddSeparableEvaluate
+{
+ public:
+  // Provide a dummy overload so the name 'Evaluate' exists for this object.
+  typename MatType::elem_type Evaluate(traits::UnconstructableType&,
+                                       const size_t,
+                                       const size_t);
+};
+
+/**
+ * Reflect the existing Evaluate().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient>
+class AddSeparableEvaluate<FunctionType, MatType, GradType,
+    HasSeparableEvaluateWithGradient, true>
+{
+ public:
+  // Reflect the existing Evaluate().
+  typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                       const size_t begin,
+                                       const size_t batchSize)
+  {
+    return static_cast<FunctionType*>(
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->Evaluate(coordinates,
+                                                          begin,
+                                                          batchSize);
+  }
+};
+
+/**
+ * If we have a separable EvaluateWithGradient() but not a separable
+ * Evaluate(), add a separable Evaluate() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableEvaluate<FunctionType, MatType, GradType, true, false>
+{
+ public:
+  /**
+   * Return the objective function for the given coordinates, starting at the
+   * given separable function using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of first function to evaluate.
+   * @param batchSize Number of functions to evaluate.
+   */
+  double Evaluate(const MatType& coordinates,
+                  const size_t begin,
+                  const size_t batchSize)
+  {
+    GradType gradient; // This will be ignored.
+    return static_cast<Function<FunctionType,
+                                MatType,
+                                GradType>*>(this)->EvaluateWithGradient(
+        coordinates, begin, gradient, batchSize);
+  }
+};
+
+/**
+ * The AddSeparableEvaluateConst mixin class will add a separable const
+ * Evaluate() method if a separable const EvaluateWithGradient() function
+ * exists, or nothing otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientConstForm>::value,
+         bool HasSeparableEvaluate =
+             traits::HasEvaluate<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateConstForm>::value>
+class AddSeparableEvaluateConst
+{
+ public:
+  // Provide a dummy overload so the name 'Evaluate' exists for this object.
+  typename MatType::elem_type Evaluate(traits::UnconstructableType&,
+                                       const size_t,
+                                       const size_t) const;
+};
+
+/**
+ * Reflect the existing Evaluate().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient>
+class AddSeparableEvaluateConst<FunctionType, MatType, GradType,
+    HasSeparableEvaluateWithGradient, true>
+{
+ public:
+  // Reflect the existing Evaluate().
+  typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                       const size_t begin,
+                                       const size_t batchSize) const
+  {
+    return static_cast<const FunctionType*>(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->Evaluate(coordinates,
+                                                                begin,
+                                                                batchSize);
+  }
+};
+
+/**
+ * If we have a separable const EvaluateWithGradient() but not a separable
+ * const Evaluate(), add a separable const Evaluate() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableEvaluateConst<FunctionType, MatType, GradType, true, false>
+{
+ public:
+  /**
+   * Return the objective function for the given coordinates, starting at the
+   * given separable function using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of first function to evaluate.
+   * @param batchSize Number of functions to evaluate.
+   */
+  typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                       const size_t begin,
+                                       const size_t batchSize) const
+  {
+    GradType gradient; // This will be ignored.
+    return static_cast<const Function<FunctionType,
+                                      MatType,
+                                      GradType>*>(this)->EvaluateWithGradient(
+        coordinates, begin, gradient, batchSize);
+  }
+};
+
+/**
+ * The AddSeparableEvaluateStatic mixin class will add a separable static
+ * Evaluate() method if a separable static EvaluateWithGradient() function
+ * exists, or nothing otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientStaticForm>::value,
+         bool HasSeparableEvaluate =
+             traits::HasEvaluate<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateStaticForm>::value>
+class AddSeparableEvaluateStatic
+{
+ public:
+  // Provide a dummy overload so the name 'Evaluate' exists for this object.
+  static typename MatType::elem_type Evaluate(traits::UnconstructableType&,
+                                              const size_t,
+                                              const size_t);
+};
+
+/**
+ * Reflect the existing Evaluate().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient>
+class AddSeparableEvaluateStatic<FunctionType, MatType, GradType,
+    HasSeparableEvaluateWithGradient, true>
+{
+ public:
+  // Reflect the existing Evaluate().
+  static typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                              const size_t begin,
+                                              const size_t batchSize)
+  {
+    return FunctionType::Evaluate(coordinates, begin, batchSize);
+  }
+};
+
+/**
+ * If we have a separable EvaluateWithGradient() but not a separable
+ * Evaluate(), add a separable Evaluate() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableEvaluateStatic<FunctionType, MatType, GradType, true,
+    false>
+{
+ public:
+  /**
+   * Return the objective function for the given coordinates, starting at the
+   * given separable function using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of first function to evaluate.
+   * @param batchSize Number of functions to evaluate.
+   */
+  static typename MatType::elem_type Evaluate(const MatType& coordinates,
+                                              const size_t begin,
+                                              const size_t batchSize)
+  {
+    GradType gradient; // This will be ignored.
+    return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
+        batchSize);
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/function/add_separable_evaluate_with_gradient.hpp

@@ -0,0 +1,308 @@
+/**
+ * @file add_separable_evaluate_with_gradient.hpp
+ * @author Ryan Curtin
+ *
+ * Adds a separable EvaluateWithGradient() function if both a separable
+ * Evaluate() and a separable Gradient() function exist.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_W_GRADIENT_HPP
+#define ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_EVALUATE_W_GRADIENT_HPP
+
+#include "traits.hpp"
+
+namespace ens {
+
+/**
+ * The AddSeparableEvaluateWithGradient mixin class will add a separable
+ * EvaluateWithGradient() method if a separable Evaluate() method and a
+ * separable Gradient() method exists, or nothing otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         // Check if there is at least one non-const Evaluate() or Gradient().
+         bool HasSeparableEvaluateGradient = traits::HasNonConstSignatures<
+             FunctionType,
+             traits::HasEvaluate,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableEvaluateForm,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableEvaluateConstForm,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableEvaluateStaticForm,
+             traits::HasGradient,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableGradientForm,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableGradientConstForm,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableGradientStaticForm>::value,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientForm>::value>
+class AddSeparableEvaluateWithGradient
+{
+ public:
+  // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
+  // object.
+  typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&,
+      const size_t,
+      const size_t);
+};
+
+/**
+ * Reflect the existing EvaluateWithGradient().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateGradient>
+class AddSeparableEvaluateWithGradient<FunctionType, MatType, GradType,
+    HasSeparableEvaluateGradient, true>
+{
+ public:
+  // Reflect the existing EvaluateWithGradient().
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   const size_t begin,
+                                                   GradType& gradient,
+                                                   const size_t batchSize)
+  {
+    return static_cast<FunctionType*>(
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->EvaluateWithGradient(
+        coordinates, begin, gradient, batchSize);
+  }
+};
+
+/**
+ * If we have a both separable Evaluate() and a separable Gradient() but
+ * not a separable EvaluateWithGradient(), add a separable
+ * EvaluateWithGradient() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableEvaluateWithGradient<FunctionType, MatType, GradType, true,
+    false>
+{
+ public:
+  /**
+   * Return both the evaluated objective function and its gradient, storing the
+   * gradient in the given matrix, starting at the given separable function
+   * and using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of separable function to begin with.
+   * @param gradient Matrix to store the gradient into.
+   * @param batchSize Number of separable functions to evaluate.
+   */
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   const size_t begin,
+                                                   GradType& gradient,
+                                                   const size_t batchSize)
+  {
+    const typename MatType::elem_type objective =
+        static_cast<Function<FunctionType, MatType, GradType>*>(this)->Evaluate(
+        coordinates, begin, batchSize);
+    static_cast<Function<FunctionType, MatType, GradType>*>(this)->Gradient(
+        coordinates, begin, gradient, batchSize);
+    return objective;
+  }
+};
+
+/**
+ * The AddSeparableEvaluateWithGradientConst mixin class will add a
+ * separable const EvaluateWithGradient() method if both a separable const
+ * Evaluate() and a separable const Gradient() function exist, or nothing
+ * otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         // Check if there is at least one const Evaluate() or Gradient().
+         bool HasSeparableEvaluateGradient = traits::HasConstSignatures<
+             FunctionType,
+             traits::HasEvaluate,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableEvaluateConstForm,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableEvaluateStaticForm,
+             traits::HasGradient,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableGradientConstForm,
+             traits::TypedForms<MatType, GradType>::template
+                 SeparableGradientStaticForm>::value,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientConstForm>::value>
+class AddSeparableEvaluateWithGradientConst
+{
+ public:
+  // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
+  // object.
+  typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&,
+      const size_t,
+      const size_t) const;
+};
+
+/**
+ * Reflect the existing EvaluateWithGradient().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateGradient>
+class AddSeparableEvaluateWithGradientConst<FunctionType, MatType, GradType,
+    HasSeparableEvaluateGradient, true>
+{
+ public:
+  // Reflect the existing Evaluate().
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   const size_t begin,
+                                                   GradType& gradient,
+                                                   const size_t batchSize) const
+  {
+    return static_cast<const FunctionType*>(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->EvaluateWithGradient(
+        coordinates, begin, gradient, batchSize);
+  }
+};
+
+/**
+ * If we have both a separable const Evaluate() and a separable const
+ * Gradient() but not a separable const EvaluateWithGradient(), add a
+ * separable const EvaluateWithGradient() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableEvaluateWithGradientConst<FunctionType, MatType, GradType,
+    true, false>
+{
+ public:
+  /**
+   * Return both the evaluated objective function and its gradient, storing the
+   * gradient in the given matrix, starting at the given separable function
+   * and using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of separable function to begin with.
+   * @param gradient Matrix to store the gradient into.
+   * @param batchSize Number of separable functions to evaluate.
+   */
+  typename MatType::elem_type EvaluateWithGradient(const MatType& coordinates,
+                                                   const size_t begin,
+                                                   GradType& gradient,
+                                                   const size_t batchSize) const
+  {
+    const typename MatType::elem_type objective =
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this)->Evaluate(coordinates,
+        begin, batchSize);
+    static_cast<const Function<FunctionType,
+                               MatType,
+                               GradType>*>(this)->Gradient(coordinates,
+        begin, gradient, batchSize);
+    return objective;
+  }
+};
+
+/**
+ * The AddSeparableEvaluateWithGradientStatic mixin class will add a
+ * separable static EvaluateWithGradient() method if both a separable
+ * static Evaluate() and a separable static gradient() function exist, or
+ * nothing otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateGradient =
+             traits::HasEvaluate<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateStaticForm>::value &&
+             traits::HasGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableGradientStaticForm>::value,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientStaticForm>::value>
+class AddSeparableEvaluateWithGradientStatic
+{
+ public:
+  // Provide a dummy overload so the name 'EvaluateWithGradient' exists for this
+  // object.
+  static typename MatType::elem_type EvaluateWithGradient(
+      traits::UnconstructableType&,
+      const size_t,
+      const size_t);
+};
+
+/**
+ * Reflect the existing EvaluateWithGradient().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateGradient>
+class AddSeparableEvaluateWithGradientStatic<FunctionType, MatType, GradType,
+    HasSeparableEvaluateGradient, true>
+{
+ public:
+  // Reflect the existing Evaluate().
+  static typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates,
+      const size_t begin,
+      GradType& gradient,
+      const size_t batchSize)
+  {
+    return FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
+        batchSize);
+  }
+};
+
+/**
+ * If we have a separable static Evaluate() and a separable static
+ * Gradient() but not a separable static EvaluateWithGradient(), add a
+ * separable static Gradient() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableEvaluateWithGradientStatic<FunctionType, MatType, GradType,
+    true, false>
+{
+ public:
+  /**
+   * Return both the evaluated objective function and its gradient, storing the
+   * gradient in the given matrix, starting at the given separable function
+   * and using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of separable function to begin with.
+   * @param gradient Matrix to store the gradient into.
+   * @param batchSize Number of separable functions to evaluate.
+   */
+  typename MatType::elem_type EvaluateWithGradient(
+      const MatType& coordinates,
+      const size_t begin,
+      GradType& gradient,
+      const size_t batchSize) const
+  {
+    const typename MatType::elem_type objective = FunctionType::Evaluate(
+        coordinates, begin, batchSize);
+    FunctionType::Gradient(coordinates, begin, gradient, batchSize);
+    return objective;
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/function/add_separable_gradient.hpp

@@ -0,0 +1,254 @@
+/**
+ * @file add_separable_gradient.hpp
+ * @author Ryan Curtin
+ *
+ * Adds a separable Gradient() function if a separable
+ * EvaluateWithGradient() function exists.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_GRADIENT_HPP
+#define ENSMALLEN_FUNCTION_ADD_DECOMPOSABLE_GRADIENT_HPP
+
+#include "traits.hpp"
+
+namespace ens {
+
+/**
+ * The AddSeparableGradient mixin class will add a separable Gradient()
+ * method if a separable EvaluateWithGradient() function exists, or nothing
+ * otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientForm>::value,
+         bool HasSeparableGradient =
+             traits::HasGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableGradientForm>::value>
+class AddSeparableGradient
+{
+ public:
+  // Provide a dummy overload so the name 'Gradient' exists for this object.
+  void Gradient(traits::UnconstructableType&, const size_t, const size_t);
+};
+
+/**
+ * Reflect the existing Gradient().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient>
+class AddSeparableGradient<FunctionType, MatType, GradType,
+    HasSeparableEvaluateWithGradient, true>
+{
+ public:
+  // Reflect the existing Gradient().
+  void Gradient(const MatType& coordinates,
+                const size_t begin,
+                GradType& gradient,
+                const size_t batchSize)
+  {
+    static_cast<FunctionType*>(
+        static_cast<Function<FunctionType,
+                             MatType,
+                             GradType>*>(this))->Gradient(
+        coordinates, begin, gradient, batchSize);
+  }
+};
+
+/**
+ * If we have a separable EvaluateWithGradient() but not a separable
+ * Gradient(), add a separable Evaluate() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableGradient<FunctionType, MatType, GradType, true, false>
+{
+ public:
+  /**
+   * Calculate the gradient and store it in the given matrix, starting at the
+   * given separable function index and using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of separable function to start at.
+   * @param gradient Matrix to store the gradient into.
+   * @param batchSize Number of separable functions to calculate for.
+   */
+  void Gradient(const MatType& coordinates,
+                const size_t begin,
+                GradType& gradient,
+                const size_t batchSize)
+  {
+    // The returned objective value will be ignored.
+    (void) static_cast<Function<FunctionType,
+                                MatType,
+                                GradType>*>(this)->EvaluateWithGradient(
+        coordinates, begin, gradient, batchSize);
+  }
+};
+
+/**
+ * The AddSeparableGradientConst mixin class will add a separable const
+ * Gradient() method if a separable const EvaluateWithGradient() function
+ * exists, or nothing otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientConstForm>::value,
+         bool HasSeparableGradient =
+             traits::HasGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableGradientConstForm>::value>
+class AddSeparableGradientConst
+{
+ public:
+  // Provide a dummy overload so the name 'Gradient' exists for this object.
+  void Gradient(traits::UnconstructableType&, const size_t, const size_t) const;
+};
+
+/**
+ * Reflect the existing Gradient().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient>
+class AddSeparableGradientConst<FunctionType, MatType, GradType,
+    HasSeparableEvaluateWithGradient, true>
+{
+ public:
+  // Reflect the existing Gradient().
+  void Gradient(const MatType& coordinates,
+                const size_t begin,
+                GradType& gradient,
+                const size_t batchSize) const
+  {
+    static_cast<const FunctionType*>(
+        static_cast<const Function<FunctionType,
+                                   MatType,
+                                   GradType>*>(this))->Gradient(coordinates,
+        begin, gradient, batchSize);
+  }
+};
+
+/**
+ * If we have a separable const EvaluateWithGradient() but not a separable
+ * const Gradient(), add a separable const Gradient() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableGradientConst<FunctionType, MatType, GradType, true, false>
+{
+ public:
+  /**
+   * Calculate the gradient and store it in the given matrix, starting at the
+   * given separable function index and using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of separable function to start at.
+   * @param gradient Matrix to store the gradient into.
+   * @param batchSize Number of separable functions to calculate for.
+   */
+  void Gradient(const MatType& coordinates,
+                const size_t begin,
+                GradType& gradient,
+                const size_t batchSize) const
+  {
+    // The returned objective value will be ignored.
+    (void) static_cast<
+        const Function<FunctionType,
+                       MatType,
+                       GradType>*>(this)->EvaluateWithGradient(
+        coordinates, begin, gradient, batchSize);
+  }
+};
+
+/**
+ * The AddSeparableEvaluateStatic mixin class will add a separable static
+ * Gradient() method if a separable static EvaluateWithGradient() function
+ * exists, or nothing otherwise.
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient =
+             traits::HasEvaluateWithGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableEvaluateWithGradientStaticForm>::value,
+         bool HasSeparableGradient =
+             traits::HasGradient<FunctionType,
+                 traits::TypedForms<MatType, GradType>::template
+                     SeparableGradientStaticForm>::value>
+class AddSeparableGradientStatic
+{
+ public:
+  // Provide a dummy overload so the name 'Gradient' exists for this object.
+  static void Gradient(traits::UnconstructableType&,
+                       const size_t,
+                       const size_t);
+};
+
+/**
+ * Reflect the existing Gradient().
+ */
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         bool HasSeparableEvaluateWithGradient>
+class AddSeparableGradientStatic<FunctionType, MatType, GradType,
+    HasSeparableEvaluateWithGradient, true>
+{
+ public:
+  // Reflect the existing Gradient().
+  static void Gradient(const MatType& coordinates,
+                       const size_t begin,
+                       GradType& gradient,
+                       const size_t batchSize)
+  {
+    FunctionType::Gradient(coordinates, begin, gradient, batchSize);
+  }
+};
+
+/**
+ * If we have a separable EvaluateWithGradient() but not a separable
+ * Gradient(), add a separable Gradient() method.
+ */
+template<typename FunctionType, typename MatType, typename GradType>
+class AddSeparableGradientStatic<FunctionType, MatType, GradType, true,
+    false>
+{
+ public:
+  /**
+   * Calculate the gradient and store it in the given matrix, starting at the
+   * given separable function index and using the given batch size.
+   *
+   * @param coordinates Coordinates to evaluate the function at.
+   * @param begin Index of separable function to start at.
+   * @param gradient Matrix to store the gradient into.
+   * @param batchSize Number of separable functions to calculate for.
+   */
+  static void Gradient(const MatType& coordinates,
+                       const size_t begin,
+                       GradType& gradient,
+                       const size_t batchSize)
+  {
+    // The returned objective value will be ignored.
+    (void) FunctionType::EvaluateWithGradient(coordinates, begin, gradient,
+        batchSize);
+  }
+};
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/function/arma_traits.hpp

@@ -0,0 +1,176 @@
+/**
+ * @file arma_traits.hpp
+ * @author Ryan Curtin
+ *
+ * Given an Armadillo type, determine its "true" base type.
+ */
+#ifndef ENSMALLEN_FUNCTION_ARMA_TRAITS_HPP
+#define ENSMALLEN_FUNCTION_ARMA_TRAITS_HPP
+
+namespace ens {
+
+/**
+ * Extract the base type of a matrix (i.e. if it is a column, return the matrix
+ * type).  If the type is unknown (or not a derived type) we just return the
+ * type itself as the typedef BaseMatType.
+ */
+
+template<typename MatType>
+struct MatTypeTraits
+{
+  typedef MatType BaseMatType;
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::Col<eT>>
+{
+  typedef arma::Mat<eT> BaseMatType;
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::Row<eT>>
+{
+  typedef arma::Mat<eT> BaseMatType;
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::SpCol<eT>>
+{
+  typedef arma::SpMat<eT> BaseMatType;
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::SpRow<eT>>
+{
+  typedef arma::SpMat<eT> BaseMatType;
+};
+
+/**
+ * Disable usage of arma::subviews and related types for optimizers.  It might
+ * be nice to also explicitly disable Armadillo expressions, but we'll hope for
+ * now nobody even tries that, since those aren't even lvalues and thus can't
+ * really work.
+ */
+
+template<typename eT>
+struct MatTypeTraits<arma::subview<eT>>
+{
+  static_assert(sizeof(arma::subview<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::subview_col<eT>>
+{
+  static_assert(sizeof(arma::subview_col<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::SpSubview<eT>>
+{
+  static_assert(sizeof(arma::SpSubview<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+
+#if ((ARMA_VERSION_MAJOR >= 10) || \
+    ((ARMA_VERSION_MAJOR == 9) && (ARMA_VERSION_MINOR >= 869)))
+
+// Armadillo 9.869+ has SpSubview_col and SpSubview_row
+
+template<typename eT>
+struct MatTypeTraits<arma::SpSubview_col<eT>>
+{
+  static_assert(sizeof(arma::SpSubview_col<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+template<typename eT>
+struct MatTypeTraits<arma::SpSubview_row<eT>>
+{
+  static_assert(sizeof(arma::SpSubview_row<eT>) == 0,
+      "Armadillo subviews cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+#endif
+
+
+template<typename eT>
+struct MatTypeTraits<arma::Cube<eT>>
+{
+  static_assert(sizeof(arma::Cube<eT>) == 0,
+      "Armadillo cubes cannot be passed to Optimize()!  Create a matrix "
+      "or a matrix alias instead!");
+};
+
+/**
+ * Issue a fatal error if the type is not an Armadillo double or floating point
+ * sparse or dense matrix.
+ */
+
+template<typename MatType>
+void RequireDenseFloatingPointType()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(sizeof(MatType) == 0,
+      "The given MatType must be arma::mat or arma::fmat or it is not known "
+      "to work!  If you would like to try anyway, set the preprocessor macro "
+      "ENS_DISABLE_TYPE_CHECKS before including ensmallen.hpp.  However, you "
+      "get to pick up all the pieces if there is a failure!");
+#endif
+}
+
+template<>
+inline void RequireDenseFloatingPointType<arma::mat>() { }
+template<>
+inline void RequireDenseFloatingPointType<arma::fmat>() { }
+
+template<typename MatType>
+void RequireFloatingPointType()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(sizeof(MatType) == 0,
+      "The given MatType must be arma::mat, arma::fmat, arma::sp_mat, or "
+      "arma::sp_fmat, or it is not known to work!  If you would like to try "
+      "anyway, set the preprocessor macro ENS_DISABLE_TYPE_CHECKS before "
+      "including ensmallen.hpp.  However, you get to pick up all the pieces if "
+      "there is a failure!");
+#endif
+}
+
+template<>
+inline void RequireFloatingPointType<arma::mat>() { }
+template<>
+inline void RequireFloatingPointType<arma::fmat>() { }
+template<>
+inline void RequireFloatingPointType<arma::sp_mat>() { }
+template<>
+inline void RequireFloatingPointType<arma::sp_fmat>() { }
+
+/**
+ * Require that the internal element type of the matrix type and gradient type
+ * are the same.  A static_assert() will fail if not.
+ */
+template<typename MatType, typename GradType>
+void RequireSameInternalTypes()
+{
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(std::is_same<typename MatType::elem_type,
+                             typename GradType::elem_type>::value,
+      "The internal element types of the given MatType and GradType must be "
+      "identical, or it is not known to work!  If you would like to try "
+      "anyway, set the preprocessor macro ENS_DISABLE_TYPE_CHECKS before "
+      "including ensmallen.hpp.  However, you get to pick up all the pieces if "
+      "there is a failure!");
+#endif
+}
+
+} // namespace ens
+
+#endif

include/ensmallen_bits/function/sfinae_utility.hpp

@@ -105,6 +105,27 @@ struct MethodFormDetector<Class, MethodForm, 7>
 //! Utility struct for checking signatures.
 template<typename U, U> struct SigCheck : std::true_type {};
 
+template<typename... Args>
+struct pack {};
+
+template<typename Func>
+struct FunctionTypes {};
+
+template<typename R, typename... A>
+struct FunctionTypes<R(A...)>
+{
+  typedef R Ret;
+  using Args = pack<A...>;
+};
+
+template<typename R, typename C, typename... A>
+struct FunctionTypes<R(C::*)(A...)>
+{
+  typedef R Ret;
+  typedef C Class;
+  using Args = pack<A...>;
+};
+
 } // namespace sfinae
 } // namespace ens
 
@@ -133,7 +154,9 @@ struct NAME                                                                    \
 <                                                                              \
   T,                                                                           \
   sig,                                                                         \
-  std::integral_constant<bool, SigCheck<sig, &T::FUNC>::value>                 \
+  std::is_same<decltype(std::declval<T>().FUNC(std::declval<                   \
+      ens::sfinae::FunctionTypes<sig>::A...>()...)),                           \
+      ens::sfinae::FunctionTypes<sig>::Ret>::type>                             \
 > : std::true_type {};
 
 /**
@@ -257,9 +280,9 @@ struct NAME                                                                  \
     ENS_HAS_METHOD_FORM_BASE(ENS_SINGLE_ARG(METHOD), ENS_SINGLE_ARG(NAME), 7)
 
 /**
- * ENS_HAS_EXACT_METHOD_FORM generates a template that allows to check at compile
- * time whether a given class has a method of the requested form. For example,
- * for the following class
+ * ENS_HAS_EXACT_METHOD_FORM generates a template that allows to check at
+ * compile time whether a given class has a method of the requested form. For
+ * example, for the following class
  *
  * class A
  * {
@@ -291,37 +314,4 @@ struct NAME                                                                  \
 #define ENS_HAS_EXACT_METHOD_FORM(METHOD, NAME) \
     ENS_HAS_METHOD_FORM_BASE(ENS_SINGLE_ARG(METHOD), ENS_SINGLE_ARG(NAME), 0)
 
-/**
- * A version of ENS_HAS_METHOD_FORM() where the maximum number of extra arguments is
- * set to the default of 7.
- *
- * ENS_HAS_METHOD_FORM generates a template that allows to check at compile time
- * whether a given class has a method of the requested form. For example, for
- * the following class
- *
- * class A
- * {
- *  public:
- *   ...
- *   Train(const arma::mat&, const arma::Row<size_t>&, double);
- *   ...
- * };
- *
- * and the following form of Train methods
- *
- * template<typename Class, typename...Ts>
- * using TrainForm =
- *     void(Class::*)(const arma::mat&, const arma::Row<size_t>&, Ts...);
- *
- * we can check whether the class A has a Train method of the specified form:
- *
- * ENS_HAS_METHOD_FORM(Train, HasTrain);
- * static_assert(HasTrain<A, TrainFrom>::value, "value should be true");
- *
- * The implementation is analogous to implementation of the macro ENS_HAS_MEM_FUNC.
- *
- * @param METHOD The name of the method to check for.
- * @param NAME The name of the struct to construct.
- */
-
 #endif

include/ensmallen_bits/function/static_checks.hpp

@@ -25,13 +25,16 @@ namespace traits {
  *
  * This is required by the FunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckEvaluate
 {
   const static bool value =
-      HasEvaluate<FunctionType, EvaluateForm>::value ||
-      HasEvaluate<FunctionType, EvaluateConstForm>::value ||
-      HasEvaluate<FunctionType, EvaluateStaticForm>::value;
+      HasEvaluate<FunctionType,
+          TypedForms<MatType, GradType>::template EvaluateForm>::value ||
+      HasEvaluate<FunctionType,
+          TypedForms<MatType, GradType>::template EvaluateConstForm>::value ||
+      HasEvaluate<FunctionType,
+          TypedForms<MatType, GradType>::template EvaluateStaticForm>::value;
 };
 
 /**
@@ -39,69 +42,84 @@ struct CheckEvaluate
  *
  * This is required by the FunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckGradient
 {
   const static bool value =
-      HasGradient<FunctionType, GradientForm>::value ||
-      HasGradient<FunctionType, GradientConstForm>::value ||
-      HasGradient<FunctionType, GradientStaticForm>::value;
+      HasGradient<FunctionType,
+          TypedForms<MatType, GradType>::template GradientForm>::value ||
+      HasGradient<FunctionType,
+          TypedForms<MatType, GradType>::template GradientConstForm>::value ||
+      HasGradient<FunctionType,
+          TypedForms<MatType, GradType>::template GradientStaticForm>::value;
 };
 
 /**
  * Check if a suitable overload of NumFunctions() is available.
  *
- * This is required by the DecomposableFunctionType API.
+ * This is required by the SeparableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckNumFunctions
 {
   const static bool value =
-      HasNumFunctions<FunctionType, NumFunctionsForm>::value ||
-      HasNumFunctions<FunctionType, NumFunctionsConstForm>::value ||
-      HasNumFunctions<FunctionType, NumFunctionsStaticForm>::value;
+      HasNumFunctions<FunctionType, TypedForms<MatType, GradType>::template
+          NumFunctionsForm>::value ||
+      HasNumFunctions<FunctionType, TypedForms<MatType, GradType>::template
+          NumFunctionsConstForm>::value ||
+      HasNumFunctions<FunctionType, TypedForms<MatType, GradType>::template
+          NumFunctionsStaticForm>::value;
 };
 
 /**
  * Check if a suitable overload of Shuffle() is available.
  *
- * This is required by the DecomposableFunctionType API.
+ * This is required by the SeparableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckShuffle
 {
   const static bool value =
-      HasShuffle<FunctionType, ShuffleForm>::value ||
-      HasShuffle<FunctionType, ShuffleConstForm>::value ||
-      HasShuffle<FunctionType, ShuffleStaticForm>::value;
+      HasShuffle<FunctionType, TypedForms<MatType, GradType>::template
+          ShuffleForm>::value ||
+      HasShuffle<FunctionType, TypedForms<MatType, GradType>::template
+          ShuffleConstForm>::value ||
+      HasShuffle<FunctionType, TypedForms<MatType, GradType>::template
+          ShuffleStaticForm>::value;
 };
 
 /**
- * Check if a suitable decomposable overload of Evaluate() is available.
+ * Check if a suitable separable overload of Evaluate() is available.
  *
- * This is required by the DecomposableFunctionType API.
+ * This is required by the SeparableFunctionType API.
  */
-template<typename FunctionType>
-struct CheckDecomposableEvaluate
+template<typename FunctionType, typename MatType, typename GradType>
+struct CheckSeparableEvaluate
 {
   const static bool value =
-      HasEvaluate<FunctionType, DecomposableEvaluateForm>::value ||
-      HasEvaluate<FunctionType, DecomposableEvaluateConstForm>::value ||
-      HasEvaluate<FunctionType, DecomposableEvaluateStaticForm>::value;
+      HasEvaluate<FunctionType, TypedForms<MatType, GradType>::template
+          SeparableEvaluateForm>::value ||
+      HasEvaluate<FunctionType, TypedForms<MatType, GradType>::template
+          SeparableEvaluateConstForm>::value ||
+      HasEvaluate<FunctionType, TypedForms<MatType, GradType>::template
+          SeparableEvaluateStaticForm>::value;
 };
 
 /**
- * Check if a suitable decomposable overload of Gradient() is available.
+ * Check if a suitable separable overload of Gradient() is available.
  *
- * This is required by the DecomposableFunctionType API.
+ * This is required by the SeparableFunctionType API.
  */
-template <typename FunctionType>
-struct CheckDecomposableGradient
+template<typename FunctionType, typename MatType, typename GradType>
+struct CheckSeparableGradient
 {
   const static bool value =
-      HasGradient<FunctionType, DecomposableGradientForm>::value ||
-      HasGradient<FunctionType, DecomposableGradientConstForm>::value ||
-      HasGradient<FunctionType, DecomposableGradientStaticForm>::value;
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SeparableGradientForm>::value ||
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SeparableGradientConstForm>::value ||
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SeparableGradientStaticForm>::value;
 };
 
 /**
@@ -109,13 +127,16 @@ struct CheckDecomposableGradient
  *
  * This is required by the ConstrainedFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckNumConstraints
 {
   const static bool value =
-      HasNumConstraints<FunctionType, NumConstraintsForm>::value ||
-      HasNumConstraints<FunctionType, NumConstraintsConstForm>::value ||
-      HasNumConstraints<FunctionType, NumConstraintsStaticForm>::value;
+      HasNumConstraints<FunctionType, TypedForms<MatType, GradType>::template
+          NumConstraintsForm>::value ||
+      HasNumConstraints<FunctionType, TypedForms<MatType, GradType>::template
+          NumConstraintsConstForm>::value ||
+      HasNumConstraints<FunctionType, TypedForms<MatType, GradType>::template
+          NumConstraintsStaticForm>::value;
 };
 
 /**
@@ -123,13 +144,19 @@ struct CheckNumConstraints
  *
  * This is required by the ConstrainedFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckEvaluateConstraint
 {
   const static bool value =
-      HasEvaluateConstraint<FunctionType, EvaluateConstraintForm>::value ||
-      HasEvaluateConstraint<FunctionType, EvaluateConstraintConstForm>::value ||
-      HasEvaluateConstraint<FunctionType, EvaluateConstraintStaticForm>::value;
+      HasEvaluateConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              EvaluateConstraintForm>::value ||
+      HasEvaluateConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              EvaluateConstraintConstForm>::value ||
+      HasEvaluateConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              EvaluateConstraintStaticForm>::value;
 };
 
 /**
@@ -137,13 +164,19 @@ struct CheckEvaluateConstraint
  *
  * This is required by the ConstrainedFunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckGradientConstraint
 {
   const static bool value =
-      HasGradientConstraint<FunctionType, GradientConstraintForm>::value ||
-      HasGradientConstraint<FunctionType, GradientConstraintConstForm>::value ||
-      HasGradientConstraint<FunctionType, GradientConstraintStaticForm>::value;
+      HasGradientConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              GradientConstraintForm>::value ||
+      HasGradientConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              GradientConstraintConstForm>::value ||
+      HasGradientConstraint<FunctionType,
+          TypedForms<MatType, GradType>::template
+              GradientConstraintStaticForm>::value;
 };
 
 /**
@@ -152,13 +185,16 @@ struct CheckGradientConstraint
  *
  * This is required by the SparseFunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckSparseGradient
 {
   const static bool value =
-      HasGradient<FunctionType, SparseGradientForm>::value ||
-      HasGradient<FunctionType, SparseGradientConstForm>::value ||
-      HasGradient<FunctionType, SparseGradientStaticForm>::value;
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SparseGradientForm>::value ||
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SparseGradientConstForm>::value ||
+      HasGradient<FunctionType, TypedForms<MatType, GradType>::template
+          SparseGradientStaticForm>::value;
 };
 
 /**
@@ -166,13 +202,16 @@ struct CheckSparseGradient
  *
  * This is required by the ResolvableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckNumFeatures
 {
   const static bool value =
-      HasNumFeatures<FunctionType, NumFeaturesForm>::value ||
-      HasNumFeatures<FunctionType, NumFeaturesConstForm>::value ||
-      HasNumFeatures<FunctionType, NumFeaturesStaticForm>::value;
+      HasNumFeatures<FunctionType, TypedForms<MatType, GradType>::template
+          NumFeaturesForm>::value ||
+      HasNumFeatures<FunctionType, TypedForms<MatType, GradType>::template
+          NumFeaturesConstForm>::value ||
+      HasNumFeatures<FunctionType, TypedForms<MatType, GradType>::template
+          NumFeaturesStaticForm>::value;
 };
 
 /**
@@ -180,13 +219,16 @@ struct CheckNumFeatures
  *
  * This is required by the ResolvableFunctionType API.
  */
-template <typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckPartialGradient
 {
   const static bool value =
-      HasPartialGradient<FunctionType, PartialGradientForm>::value ||
-      HasPartialGradient<FunctionType, PartialGradientConstForm>::value ||
-      HasPartialGradient<FunctionType, PartialGradientStaticForm>::value;
+      HasPartialGradient<FunctionType, TypedForms<MatType, GradType>::template
+          PartialGradientForm>::value ||
+      HasPartialGradient<FunctionType, TypedForms<MatType, GradType>::template
+          PartialGradientConstForm>::value ||
+      HasPartialGradient<FunctionType, TypedForms<MatType, GradType>::template
+          PartialGradientStaticForm>::value;
 };
 
 /**
@@ -194,207 +236,239 @@ struct CheckPartialGradient
  *
  * This is required by the FunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 struct CheckEvaluateWithGradient
 {
   const static bool value =
-      HasEvaluateWithGradient<FunctionType, EvaluateWithGradientForm>::value ||
       HasEvaluateWithGradient<FunctionType,
-          EvaluateWithGradientConstForm>::value ||
+          TypedForms<MatType, GradType>::template
+              EvaluateWithGradientForm>::value ||
       HasEvaluateWithGradient<FunctionType,
-          EvaluateWithGradientStaticForm>::value;
+          TypedForms<MatType, GradType>::template
+              EvaluateWithGradientConstForm>::value ||
+      HasEvaluateWithGradient<FunctionType,
+          TypedForms<MatType, GradType>::template
+              EvaluateWithGradientStaticForm>::value;
 };
 
 /**
- * Check if a suitable decomposable overload of EvaluateWithGradient() is
+ * Check if a suitable separable overload of EvaluateWithGradient() is
  * available.
  *
  * This is required by the FunctionType API.
  */
-template<typename FunctionType>
-struct CheckDecomposableEvaluateWithGradient
+template<typename FunctionType, typename MatType, typename GradType>
+struct CheckSeparableEvaluateWithGradient
 {
   const static bool value =
       HasEvaluateWithGradient<FunctionType,
-          DecomposableEvaluateWithGradientForm>::value ||
+          TypedForms<MatType, GradType>::template
+              SeparableEvaluateWithGradientForm>::value ||
       HasEvaluateWithGradient<FunctionType,
-          DecomposableEvaluateWithGradientConstForm>::value ||
+          TypedForms<MatType, GradType>::template
+              SeparableEvaluateWithGradientConstForm>::value ||
       HasEvaluateWithGradient<FunctionType,
-          DecomposableEvaluateWithGradientStaticForm>::value;
+          TypedForms<MatType, GradType>::template
+              SeparableEvaluateWithGradientStaticForm>::value;
 };
 
 /**
  * Perform checks for the regular FunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckFunctionTypeAPI()
 {
-  static_assert(CheckEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckEvaluate<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the FunctionType API; see the optimizer tutorial for details.");
 
-  static_assert(CheckGradient<FunctionType>::value,
+  static_assert(CheckGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Gradient(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the FunctionType API; see the optimizer tutorial for details.");
 
-  static_assert(CheckEvaluateWithGradient<FunctionType>::value,
+  static_assert(
+      CheckEvaluateWithGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of "
       "EvaluateWithGradient().  Please check that the FunctionType fully "
       "satisfies the requirements of the FunctionType API; see the optimizer "
       "tutorial for more details.");
+#endif
 }
 
 /**
- * Perform checks for the DecomposableFunctionType API.
+ * Perform checks for the SeparableFunctionType API.
  */
-template<typename FunctionType>
-inline void CheckDecomposableFunctionTypeAPI()
+template<typename FunctionType, typename MatType, typename GradType>
+inline void CheckSeparableFunctionTypeAPI()
 {
-  static_assert(CheckDecomposableEvaluate<FunctionType>::value,
-      "The FunctionType does not have a correct definition of a decomposable "
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckSeparableEvaluate<FunctionType,
+                                       MatType,
+                                       GradType>::value,
+      "The FunctionType does not have a correct definition of a separable "
       "Evaluate() method.  Please check that the FunctionType fully satisfies"
-      " the requirements of the DecomposableFunctionType API; see the optimizer"
+      " the requirements of the SeparableFunctionType API; see the optimizer"
       " tutorial for more details.");
 
-  static_assert(CheckDecomposableGradient<FunctionType>::value,
-      "The FunctionType does not have a correct definition of a decomposable "
+  static_assert(CheckSeparableGradient<FunctionType,
+                                       MatType,
+                                       GradType>::value,
+      "The FunctionType does not have a correct definition of a separable "
       "Gradient() method.  Please check that the FunctionType fully satisfies"
-      " the requirements of the DecomposableFunctionType API; see the optimizer"
+      " the requirements of the SeparableFunctionType API; see the optimizer"
       " tutorial for more details.");
 
-  static_assert(CheckDecomposableEvaluateWithGradient<FunctionType>::value,
-      "The FunctionType does not have a correct definition of a decomposable "
+  static_assert(CheckSeparableEvaluateWithGradient<FunctionType,
+                                                   MatType,
+                                                   GradType>::value,
+      "The FunctionType does not have a correct definition of a separable "
       "EvaluateWithGradient() method.  Please check that the FunctionType "
-      "fully satisfies the requirements of the DecomposableFunctionType API; "
+      "fully satisfies the requirements of the SeparableFunctionType API; "
       "see the optimizer tutorial for more details.");
 
-  static_assert(CheckNumFunctions<FunctionType>::value,
+  static_assert(CheckNumFunctions<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of NumFunctions(). "
       "Please check that the FunctionType fully satisfies the requirements of "
-      "the DecomposableFunctionType API; see the optimizer tutorial for more "
+      "the SeparableFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckShuffle<FunctionType>::value,
+  static_assert(CheckShuffle<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Shuffle(). "
       "Please check that the FunctionType fully satisfies the requirements of "
-      "the DecomposableFunctionType API; see the optimizer tutorial for more "
+      "the SeparableFunctionType API; see the optimizer tutorial for more "
       "details.");
+#endif
 }
 
 /**
  * Perform checks for the SparseFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckSparseFunctionTypeAPI()
 {
-  static_assert(CheckNumFunctions<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckNumFunctions<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of NumFunctions(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the SparseFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckDecomposableEvaluate<FunctionType>::value,
+  static_assert(CheckSeparableEvaluate<FunctionType,
+                                          MatType,
+                                          GradType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the SparseFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckSparseGradient<FunctionType>::value,
+  static_assert(CheckSparseGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of a sparse "
       "Gradient() method. Please check that the FunctionType fully satisfies "
       "the requirements of the SparseFunctionType API; see the optimizer "
       "tutorial for more details.");
+#endif
 }
 
 /**
- * Perform checks for the NonDifferentiableFunctionType API.
+ * Perform checks for the ArbitraryFunctionType API.
  */
-template<typename FunctionType>
-inline void CheckNonDifferentiableFunctionTypeAPI()
+template<typename FunctionType, typename MatType>
+inline void CheckArbitraryFunctionTypeAPI()
 {
-  static_assert(CheckEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckEvaluate<FunctionType, MatType, MatType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
-      "the NonDifferentiableFunctionType API; see the optimizer tutorial for "
+      "the ArbitraryFunctionType API; see the optimizer tutorial for "
       "more details.");
+#endif
 }
 
 /**
  * Perform checks for the ResolvableFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckResolvableFunctionTypeAPI()
 {
-  static_assert(CheckNumFeatures<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckNumFeatures<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of NumFeatures(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the ResolvableFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckEvaluate<FunctionType>::value,
+  static_assert(CheckEvaluate<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the ResolvableFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckPartialGradient<FunctionType>::value,
+  static_assert(CheckPartialGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of a partial "
       "Gradient() function. Please check that the FunctionType fully satisfies "
       "the requirements of the ResolvableFunctionType API; see the optimizer "
       "tutorial for more details.");
+#endif
 }
 
 /**
  * Perform checks for the ConstrainedFunctionType API.
  */
-template<typename FunctionType>
+template<typename FunctionType, typename MatType, typename GradType>
 inline void CheckConstrainedFunctionTypeAPI()
 {
-  static_assert(CheckEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckEvaluate<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckGradient<FunctionType>::value,
+  static_assert(CheckGradient<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of Gradient(). "
       "Please check that the FunctionType fully satisfies the requirements of "
       "the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckNumConstraints<FunctionType>::value,
+  static_assert(CheckNumConstraints<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of NumConstraints()."
       " Please check that the FunctionType fully satisfies the requirements of "
       "the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckEvaluateConstraint<FunctionType>::value,
+  static_assert(CheckEvaluateConstraint<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of "
       "EvaluateConstraint(). Please check that the FunctionType fully satisfies"
       " the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
 
-  static_assert(CheckGradientConstraint<FunctionType>::value,
+  static_assert(CheckGradientConstraint<FunctionType, MatType, GradType>::value,
       "The FunctionType does not have a correct definition of "
       "GradientConstraint(). Please check that the FunctionType fully satisfies"
       " the ConstrainedFunctionType API; see the optimizer tutorial for more "
       "details.");
+#endif
 }
 
 /**
- * Perform checks for the NonDifferentiableDecomposableFunctionType API.  (I
+ * Perform checks for the ArbitrarySeparableFunctionType API.  (I
  * know, it is a long name...)
  */
-template<typename FunctionType>
-inline void CheckNonDifferentiableDecomposableFunctionTypeAPI()
+template<typename FunctionType, typename MatType>
+inline void CheckArbitrarySeparableFunctionTypeAPI()
 {
-  static_assert(CheckDecomposableEvaluate<FunctionType>::value,
+#ifndef ENS_DISABLE_TYPE_CHECKS
+  static_assert(CheckSeparableEvaluate<FunctionType,
+                                          MatType,
+                                          MatType>::value,
       "The FunctionType does not have a correct definition of Evaluate(). "
       "Please check that the FunctionType fully satisfies the requirements of "
-      "the NonDifferentiableDecomposableFunctionType API; see the optimizer "
+      "the ArbitrarySeparableFunctionType API; see the optimizer "
       "tutorial for more details.");
+#endif
 }
 
 } // namespace traits

include/ensmallen_bits/function/traits.hpp

@@ -14,6 +14,7 @@
 #define ENSMALLEN_FUNCTION_TRAITS_HPP
 
 #include "sfinae_utility.hpp"
+#include "arma_traits.hpp"
 
 namespace ens {
 namespace traits {
@@ -38,211 +39,253 @@ ENS_HAS_EXACT_METHOD_FORM(GradientConstraint, HasGradientConstraint)
 ENS_HAS_EXACT_METHOD_FORM(NumFeatures, HasNumFeatures)
 //! Detect a PartialGradient() method.
 ENS_HAS_EXACT_METHOD_FORM(PartialGradient, HasPartialGradient)
+//! Detect an MaxIterations() method.
+ENS_HAS_EXACT_METHOD_FORM(MaxIterations, HasMaxIterations)
+//! Detect an ResetPolicy() method.
+ENS_HAS_EXACT_METHOD_FORM(ResetPolicy, HasResetPolicy)
+//! Detect an BatchSize() method.
+ENS_HAS_EXACT_METHOD_FORM(BatchSize, HasBatchSize)
 
-//! This is the form of a non-const Evaluate() method.
-template<typename FunctionType>
-using EvaluateForm = double(FunctionType::*)(const arma::mat&);
+template<typename MatType, typename GradType>
+struct TypedForms
+{
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
 
-//! This is the form of a const Evaluate() method.
-template<typename FunctionType>
-using EvaluateConstForm =
-    double(FunctionType::*)(const arma::mat&) const;
+  //! This is the form of a non-const Evaluate() method.
+  template<typename FunctionType>
+  using EvaluateForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&);
 
-//! This is the form of a static Evaluate() method.
-template<typename FunctionType>
-using EvaluateStaticForm = double(*)(const arma::mat&);
+  //! This is the form of a const Evaluate() method.
+  template<typename FunctionType>
+  using EvaluateConstForm = typename BaseMatType::elem_type(FunctionType::*)(
+      const BaseMatType&) const;
 
-//! This is the form of a non-const Gradient() method.
-template<typename FunctionType>
-using GradientForm = void(FunctionType::*)(const arma::mat&, arma::mat&);
+  //! This is the form of a static Evaluate() method.
+  template<typename FunctionType>
+  using EvaluateStaticForm = typename BaseMatType::elem_type(*)(
+      const BaseMatType&);
 
-//! This is the form of a const Gradient() method.
-template<typename FunctionType>
-using GradientConstForm =
-    void(FunctionType::*)(const arma::mat&, arma::mat&) const;
+  //! This is the form of a non-const Gradient() method.
+  template<typename FunctionType>
+  using GradientForm = void(FunctionType::*)(const BaseMatType&, BaseGradType&);
 
-//! This is the form of a static Gradient() method.
-template<typename FunctionType>
-using GradientStaticForm = void(*)(const arma::mat&, arma::mat&);
+  //! This is the form of a const Gradient() method.
+  template<typename FunctionType>
+  using GradientConstForm =
+      void(FunctionType::*)(const BaseMatType&, BaseGradType&) const;
 
-//! This is the form of a non-const EvaluateWithGradient() method.
-template<typename FunctionType>
-using EvaluateWithGradientForm =
-    double(FunctionType::*)(const arma::mat&, arma::mat&);
+  //! This is the form of a static Gradient() method.
+  template<typename FunctionType>
+  using GradientStaticForm = void(*)(const BaseMatType&, BaseGradType&);
 
-//! This is the form of a const EvaluateWithGradient() method.
-template<typename FunctionType>
-using EvaluateWithGradientConstForm =
-    double(FunctionType::*)(const arma::mat&, arma::mat&) const;
+  //! This is the form of a non-const EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using EvaluateWithGradientForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       BaseGradType&);
 
-//! This is the form of a static EvaluateWithGradient() method.
-template<typename FunctionType>
-using EvaluateWithGradientStaticForm =
-    double(*)(const arma::mat&, arma::mat&);
+  //! This is the form of a const EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using EvaluateWithGradientConstForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       BaseGradType&) const;
 
-//! This is the form of a non-const NumFunctions() method.
-template <typename FunctionType>
-using NumFunctionsForm = size_t(FunctionType::*)();
+  //! This is the form of a static EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using EvaluateWithGradientStaticForm = typename BaseMatType::elem_type(*)(
+      const BaseMatType&, BaseGradType&);
 
-//! This is the form of a const NumFunctions() method.
-template <typename FunctionType>
-using NumFunctionsConstForm = size_t(FunctionType::*)() const;
+  //! This is the form of a non-const NumFunctions() method.
+  template <typename FunctionType>
+  using NumFunctionsForm = size_t(FunctionType::*)();
 
-//! This is the form of a static NumFunctions() method.
-template<typename FunctionType>
-using NumFunctionsStaticForm = size_t(*)();
+  //! This is the form of a const NumFunctions() method.
+  template <typename FunctionType>
+  using NumFunctionsConstForm = size_t(FunctionType::*)() const;
 
-//! This is the form of a non-const Shuffle() method.
-template<typename FunctionType>
-using ShuffleForm = void(FunctionType::*)();
+  //! This is the form of a static NumFunctions() method.
+  template<typename FunctionType>
+  using NumFunctionsStaticForm = size_t(*)();
 
-//! This is the form of a const Shuffle() method.
-template<typename FunctionType>
-using ShuffleConstForm = void(FunctionType::*)() const;
+  //! This is the form of a non-const Shuffle() method.
+  template<typename FunctionType>
+  using ShuffleForm = void(FunctionType::*)();
 
-//! This is the form of a static Shuffle() method.
-template<typename FunctionType>
-using ShuffleStaticForm = void(*)();
+  //! This is the form of a const Shuffle() method.
+  template<typename FunctionType>
+  using ShuffleConstForm = void(FunctionType::*)() const;
 
-//! This is the form of a decomposable Evaluate() method.
-template<typename FunctionType>
-using DecomposableEvaluateForm = double(FunctionType::*)(
-    const arma::mat&, const size_t, const size_t);
+  //! This is the form of a static Shuffle() method.
+  template<typename FunctionType>
+  using ShuffleStaticForm = void(*)();
 
-//! This is the form of a decomposable const Evaluate() method.
-template<typename FunctionType>
-using DecomposableEvaluateConstForm = double(FunctionType::*)(
-    const arma::mat&, const size_t, const size_t) const;
+  //! This is the form of a separable Evaluate() method.
+  template<typename FunctionType>
+  using SeparableEvaluateForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       const size_t,
+                                                       const size_t);
 
-//! This is the form of a decomposable static Evaluate() method.
-template<typename FunctionType>
-using DecomposableEvaluateStaticForm = double(*)(
-    const arma::mat&, const size_t, const size_t);
+  //! This is the form of a separable const Evaluate() method.
+  template<typename FunctionType>
+  using SeparableEvaluateConstForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       const size_t,
+                                                       const size_t) const;
 
-//! This is the form of a decomposable non-const Gradient() method.
-template<typename FunctionType>
-using DecomposableGradientForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t);
+  //! This is the form of a separable static Evaluate() method.
+  template<typename FunctionType>
+  using SeparableEvaluateStaticForm = typename BaseMatType::elem_type(*)(
+        const BaseMatType&, const size_t, const size_t);
 
-//! This the form of a decomposable const Gradient() method.
-template<typename FunctionType>
-using DecomposableGradientConstForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t) const;
+  //! This is the form of a separable non-const Gradient() method.
+  template<typename FunctionType>
+  using SeparableGradientForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t);
 
-//! This is the form of a decomposable static Gradient() method.
-template<typename FunctionType>
-using DecomposableGradientStaticForm = void(*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t);
+  //! This the form of a separable const Gradient() method.
+  template<typename FunctionType>
+  using SeparableGradientConstForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t) const;
 
-//! This is the form of a decomposable non-const EvaluateWithGradient() method.
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientForm = double(FunctionType::*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t);
+  //! This is the form of a separable static Gradient() method.
+  template<typename FunctionType>
+  using SeparableGradientStaticForm = void(*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t);
 
-//! This is the form of a decomposable const EvaluateWithGradient() method.
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientConstForm = double(FunctionType::*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t) const;
+  //! This is the form of a separable non-const EvaluateWithGradient()
+  //! method.
+  template<typename FunctionType>
+  using SeparableEvaluateWithGradientForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       const size_t,
+                                                       BaseGradType&,
+                                                       const size_t);
 
-//! This is the form of a decomposable static EvaluateWithGradient() method.
-template<typename FunctionType>
-using DecomposableEvaluateWithGradientStaticForm = double(*)(
-    const arma::mat&, const size_t, arma::mat&, const size_t);
+  //! This is the form of a separable const EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using SeparableEvaluateWithGradientConstForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const BaseMatType&,
+                                                       const size_t,
+                                                       BaseGradType&,
+                                                       const size_t) const;
 
-//! This is the form of a non-const NumConstraints() method.
-template<typename FunctionType>
-using NumConstraintsForm = size_t(FunctionType::*)();
+  //! This is the form of a separable static EvaluateWithGradient() method.
+  template<typename FunctionType>
+  using SeparableEvaluateWithGradientStaticForm =
+      typename BaseMatType::elem_type(*)(const BaseMatType&,
+                                         const size_t,
+                                         BaseGradType&,
+                                         const size_t);
 
-//! This is the form of a const NumConstraints() method.
-template<typename FunctionType>
-using NumConstraintsConstForm = size_t(FunctionType::*)() const;
+  //! This is the form of a non-const NumConstraints() method.
+  template<typename FunctionType>
+  using NumConstraintsForm = size_t(FunctionType::*)();
 
-//! This is the form of a static NumConstraints() method.
-template<typename FunctionType>
-using NumConstraintsStaticForm = size_t(*)();
+  //! This is the form of a const NumConstraints() method.
+  template<typename FunctionType>
+  using NumConstraintsConstForm = size_t(FunctionType::*)() const;
 
-//! This is the form of a non-const EvaluateConstraint() method.
-template <typename FunctionType>
-using EvaluateConstraintForm = double(FunctionType::*)(
-    const size_t, const arma::mat&);
+  //! This is the form of a static NumConstraints() method.
+  template<typename FunctionType>
+  using NumConstraintsStaticForm = size_t(*)();
 
-//! This is the form of a const EvaluateConstraint() method.
-template<typename FunctionType>
-using EvaluateConstraintConstForm = double(FunctionType::*)(
-    const size_t, const arma::mat&) const;
+  //! This is the form of a non-const EvaluateConstraint() method.
+  template <typename FunctionType>
+  using EvaluateConstraintForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const size_t,
+                                                       const BaseMatType&);
 
-//! This is the form of a static EvaluateConstraint() method.
-template<typename FunctionType>
-using EvaluateConstraintStaticForm = double(*)(const size_t, const arma::mat&);
+  //! This is the form of a const EvaluateConstraint() method.
+  template<typename FunctionType>
+  using EvaluateConstraintConstForm =
+      typename BaseMatType::elem_type(FunctionType::*)(const size_t,
+                                                       const BaseMatType&)
+          const;
 
-//! This is the form of a non-const GradientConstraint() method.
-template <typename FunctionType>
-using GradientConstraintForm = void(FunctionType::*)(
-    const size_t, const arma::mat&, arma::mat&);
+  //! This is the form of a static EvaluateConstraint() method.
+  template<typename FunctionType>
+  using EvaluateConstraintStaticForm = typename BaseMatType::elem_type(*)(
+      const size_t, const BaseMatType&);
 
-//! This is the form of a const GradientConstraint() method.
-template<typename FunctionType>
-using GradientConstraintConstForm = void(FunctionType::*)(
-    const size_t, const arma::mat&, arma::mat&) const;
+  //! This is the form of a non-const GradientConstraint() method.
+  template <typename FunctionType>
+  using GradientConstraintForm = void(FunctionType::*)(
+      const size_t, const BaseMatType&, BaseGradType&);
 
-//! This is the form of a static GradientConstraint() method.
-template<typename Class, typename... Ts>
-using GradientConstraintStaticForm = void(*)(
-    const size_t, const arma::mat&, arma::mat&);
+  //! This is the form of a const GradientConstraint() method.
+  template<typename FunctionType>
+  using GradientConstraintConstForm = void(FunctionType::*)(
+      const size_t, const BaseMatType&, BaseGradType&) const;
 
-//! This is the form of a non-const sparse Gradient() method.
-template<typename FunctionType>
-using SparseGradientForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::sp_mat&, const size_t);
+  //! This is the form of a static GradientConstraint() method.
+  template<typename Class, typename... Ts>
+  using GradientConstraintStaticForm = void(*)(
+      const size_t, const BaseMatType&, BaseGradType&);
 
-//! This is the form of a const sparse Gradient() method.
-template<typename FunctionType>
-using SparseGradientConstForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::sp_mat&, const size_t) const;
+  //! This is the form of a non-const sparse Gradient() method.
+  //! This check isn't particularly useful---the user needs to specify a sparse
+  //! gradient type...
+  template<typename FunctionType>
+  using SparseGradientForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t);
 
-//! This is the form of a static sparse Gradient() method.
-template<typename FunctionType>
-using SparseGradientStaticForm = void(*)(
-    const arma::mat&, const size_t, arma::sp_mat&, const size_t);
+  //! This is the form of a const sparse Gradient() method.
+  //! This check isn't particularly useful---the user needs to specify a sparse
+  //! gradient type...
+  template<typename FunctionType>
+  using SparseGradientConstForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t) const;
 
-//! This is the form of a non-const NumFeatures() method.
-template<typename FunctionType>
-using NumFeaturesForm = size_t(FunctionType::*)();
+  //! This is the form of a static sparse Gradient() method.
+  //! This check isn't particularly useful---the user needs to specify a sparse
+  //! gradient type...
+  template<typename FunctionType>
+  using SparseGradientStaticForm = void(*)(
+      const BaseMatType&, const size_t, BaseGradType&, const size_t);
 
-//! This is the form of a const NumFeatures() method.
-template<typename FunctionType>
-using NumFeaturesConstForm = size_t(FunctionType::*)() const;
+  //! This is the form of a non-const NumFeatures() method.
+  template<typename FunctionType>
+  using NumFeaturesForm = size_t(FunctionType::*)();
 
-//! This is the form of a static NumFeatures() method.
-template<typename FunctionType>
-using NumFeaturesStaticForm = size_t(*)();
+  //! This is the form of a const NumFeatures() method.
+  template<typename FunctionType>
+  using NumFeaturesConstForm = size_t(FunctionType::*)() const;
 
-//! This is the form of a non-const PartialGradient() method.
-template<typename FunctionType>
-using PartialGradientForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::sp_mat&);
+  //! This is the form of a static NumFeatures() method.
+  template<typename FunctionType>
+  using NumFeaturesStaticForm = size_t(*)();
 
-//! This is the form of a const PartialGradient() method.
-template<typename FunctionType>
-using PartialGradientConstForm = void(FunctionType::*)(
-    const arma::mat&, const size_t, arma::sp_mat&) const;
+  //! This is the form of a non-const PartialGradient() method.
+  template<typename FunctionType>
+  using PartialGradientForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&);
 
-//! This is the form of a static PartialGradient() method.
-template<typename FunctionType>
-using PartialGradientStaticForm = void(*)(
-    const arma::mat&, const size_t, arma::sp_mat&);
+  //! This is the form of a const PartialGradient() method.
+  template<typename FunctionType>
+  using PartialGradientConstForm = void(FunctionType::*)(
+      const BaseMatType&, const size_t, BaseGradType&) const;
 
-//! This is a utility struct that will match any non-const form.
-template<typename FunctionType, typename... Ts>
-using OtherForm = double(FunctionType::*)(Ts...);
+  //! This is the form of a static PartialGradient() method.
+  template<typename FunctionType>
+  using PartialGradientStaticForm = void(*)(
+      const BaseMatType&, const size_t, BaseGradType&);
 
-//! This is a utility struct that will match any const form.
-template<typename FunctionType, typename... Ts>
-using OtherConstForm = double(FunctionType::*)(Ts...) const;
+  //! This is a utility struct that will match any non-const form.
+  template<typename FunctionType, typename... Ts>
+  using OtherForm = typename BaseMatType::elem_type(FunctionType::*)(Ts...);
 
-//! This is a utility struct that will match any static form.
-template<typename FunctionType, typename... Ts>
-using OtherStaticForm = double(*)(Ts...);
+  //! This is a utility struct that will match any const form.
+  template<typename FunctionType, typename... Ts>
+  using OtherConstForm = typename BaseMatType::elem_type(FunctionType::*)(Ts...)
+      const;
+
+  //! This is a utility struct that will match any static form.
+  template<typename FunctionType, typename... Ts>
+  using OtherStaticForm = typename BaseMatType::elem_type(*)(Ts...);
+};
 
 /**
  * This is a utility type used to provide unusable overloads from each of the
@@ -299,12 +342,12 @@ struct HasNonConstSignatures
 /**
  * Utility struct: sometimes we want to know if we have two functions available,
  * and that at least one of them is const and both of them are not non-const and
- * non-static.  If the corresponding checkers (from ENS_HAS_METHOD_FORM()) are given
- * as CheckerA and CheckerB, and the corresponding const and static function
- * signatures are given as ConstSignatureA, StaticSignatureA, ConstSignatureB,
- * and StaticSignatureB, then 'value' will be true if methods with the correct
- * names exist in the given ClassType and at least one of those two methods is
- * const, and neither method is non-const and non-static.
+ * non-static.  If the corresponding checkers (from ENS_HAS_METHOD_FORM()) are
+ * given as CheckerA and CheckerB, and the corresponding const and static
+ * function signatures are given as ConstSignatureA, StaticSignatureA,
+ * ConstSignatureB, and StaticSignatureB, then 'value' will be true if methods
+ * with the correct names exist in the given ClassType and at least one of those
+ * two methods is const, and neither method is non-const and non-static.
  */
 template<typename ClassType,
          template<typename, template<typename...> class, size_t> class CheckerA,
@@ -332,6 +375,60 @@ struct HasConstSignatures
   const static bool value = HasEitherConstForm && HasAnyFormA && HasAnyFormB;
 };
 
+//! Utility struct, check if size_t BatchSize() const or size_t BatchSize()
+//! exists.
+template<typename OptimizerType>
+struct HasBatchSizeSignature
+{
+  template<typename C>
+  using BatchSizeConstForm = size_t(C::*)(void) const;
+
+  template<typename C>
+  using BatchSizeForm = size_t(C::*)(void);
+
+  const static bool value =
+      HasBatchSize<OptimizerType, BatchSizeForm>::value ||
+      HasBatchSize<OptimizerType, BatchSizeConstForm>::value;
+};
+
+//! Utility struct, check if size_t MaxIterations() const exists.
+template<typename OptimizerType>
+struct HasMaxIterationsSignature
+{
+  template<typename C>
+  using HasMaxIterationsForm = size_t(C::*)(void) const;
+
+  const static bool value =
+      HasMaxIterations<OptimizerType, HasMaxIterationsForm>::value;
+};
+
+//! Utility struct, check if size_t NumFunctions() const or
+//! size_t NumFunctions() exists.
+template<typename OptimizerType>
+struct HasNumFunctionsSignature
+{
+  template<typename C>
+  using NumFunctionsConstForm = size_t(C::*)(void) const;
+
+  template<typename C>
+  using NumFunctionsForm = size_t(C::*)(void);
+
+  const static bool value =
+      HasNumFunctions<OptimizerType, NumFunctionsForm>::value ||
+      HasNumFunctions<OptimizerType, NumFunctionsConstForm>::value;
+};
+
+//! Utility struct, check if bool ResetPolicy() exists.
+template<typename OptimizerType>
+struct HasResetPolicySignature
+{
+  template<typename C>
+  using HasResetPolicyForm = bool&(C::*)(void);
+
+  const static bool value =
+      HasResetPolicy<OptimizerType, HasResetPolicyForm>::value;
+};
+
 } // namespace traits
 } // namespace ens
 

include/ensmallen_bits/fw/atoms.hpp

@@ -18,7 +18,8 @@ namespace ens {
 
 /**
  * Class to hold the information and operations of current atoms in the
- * soluton space.
+ * soluton space.  This is not fully templatized, and may cost some extra
+ * operations for the conversion.
  */
 class Atoms
 {
@@ -31,7 +32,7 @@ class Atoms
    * @param v new atom to be added.
    * @param c coefficient of the new atom.
    */
-  void AddAtom(const arma::vec& v, FuncSq& function, const double c = 0)
+  void AddAtom(const arma::mat& v, FuncSq& function, const double c = 0)
   {
     if (currentAtoms.is_empty())
     {
@@ -89,7 +90,7 @@ class Atoms
       // Solve for current gradient.
       arma::mat x;
       RecoverVector(x);
-      arma::mat gradient(size(x));
+      arma::mat gradient(arma::size(x));
       function.Gradient(x, gradient);
 
       // Find possible atom to be deleted.
@@ -113,7 +114,7 @@ class Atoms
       // add an atom norm constraint, you could use projected gradient method,
       // see the implementaton of ProjectedGradientEnhancement().
       arma::vec newCoeffs =
-          solve(function.MatrixA() * newAtoms, function.Vectorb());
+          solve(function.MatrixA() * newAtoms, function.Vectorb(), arma::solve_opts::fast);
 
       // Evaluate the function again.
       double Fnew = function.Evaluate(newAtoms * newCoeffs);

include/ensmallen_bits/fw/constr_lpball.hpp

@@ -79,36 +79,42 @@ class ConstrLpBallSolver
    * @param v Input local gradient.
    * @param s Output optimal solution in the constrained domain (lp ball).
    */
-  void Optimize(const arma::mat& v,
-                arma::mat& s)
+  template<typename MatType>
+  void Optimize(const MatType& v,
+                MatType& s)
   {
+    typedef typename MatType::elem_type ElemType;
+
     if (p == std::numeric_limits<double>::infinity())
     {
       // l-inf ball.
-      s = -sign(v);
+      s = -arma::sign(v);
       if (regFlag)
-        s = s / lambda;   // element-wise division.
+      {
+        // Do element-wise division.
+        s /= arma::conv_to<arma::Col<ElemType>>::from(lambda);
+      }
     }
     else if (p > 1.0)
     {
       // lp ball with 1<p<inf.
       if (regFlag)
-        s = v / lambda;
+        s = v / arma::conv_to<arma::Col<ElemType>>::from(lambda);
       else
         s = v;
 
       double q = 1 / (1.0 - 1.0 / p);
-      s = - sign(v) % pow(abs(s), q - 1);  // element-wise multiplication.
+      s = -arma::sign(v) % arma::pow(arma::abs(s), q - 1);
       s = arma::normalise(s, p);
 
       if (regFlag)
-        s = s / lambda;
+        s = s / arma::conv_to<arma::Col<ElemType>>::from(lambda);
     }
     else if (p == 1.0)
     {
       // l1 ball, also used in OMP.
       if (regFlag)
-        s = arma::abs(v / lambda);
+        s = arma::abs(v / arma::conv_to<arma::Col<ElemType>>::from(lambda));
       else
         s = arma::abs(v);
 
@@ -119,7 +125,7 @@ class ConstrLpBallSolver
       s(k) = -((0.0 < v(k)) - (v(k) < 0.0));
 
       if (regFlag)
-        s = s / lambda;
+        s = s / arma::conv_to<arma::Col<ElemType>>::from(lambda);
     }
     else
     {
@@ -135,14 +141,14 @@ class ConstrLpBallSolver
   double& P() { return p;}
 
   //! Get regularization flag.
-  bool RegFlag() const {return regFlag;}
+  bool RegFlag() const { return regFlag; }
   //! Modify regularization flag.
-  bool& RegFlag() {return regFlag;}
+  bool& RegFlag() { return regFlag; }
 
   //! Get the regularization parameter.
-  arma::vec Lambda() const {return lambda;}
+  arma::vec Lambda() const { return lambda; }
   //! Modify the regularization parameter.
-  arma::vec& Lambda() {return lambda;}
+  arma::vec& Lambda() { return lambda; }
 
  private:
   //! lp norm, 1<=p<=inf;

include/ensmallen_bits/fw/constr_structure_group.hpp

@@ -73,7 +73,7 @@ class ConstrStructGroupSolver
    *                       group, and compute norm in each group.
    */
   ConstrStructGroupSolver(GroupType& groupExtractor) :
-    groupExtractor(groupExtractor)
+      groupExtractor(groupExtractor)
   { /* Nothing to do */ }
 
   /**
@@ -82,18 +82,21 @@ class ConstrStructGroupSolver
    * @param v Input local gradient.
    * @param s Output optimal solution in the constrained atom domain.
    */
-  void Optimize(const arma::mat& v, arma::mat& s)
+  template<typename MatType>
+  void Optimize(const MatType& v, MatType& s)
   {
+    typedef typename MatType::elem_type ElemType;
+
     size_t nGroups = groupExtractor.NumGroups();
-    double dualNorm = 0;
+    ElemType dualNorm = 0;
     size_t optimalGroup = 1;
 
     // Find the optimal group.
     for (size_t i = 1; i <= nGroups; ++i)
     {
-      arma::vec y;
+      MatType y;
       groupExtractor.ProjectToGroup(v, i, y);
-      double newNorm = groupExtractor.DualNorm(y, i);
+      ElemType newNorm = groupExtractor.DualNorm(y, i);
 
       // Find the group with largest dual norm.
       if (newNorm > dualNorm)
@@ -142,11 +145,12 @@ class GroupLpBall
    * @param groupId input ID number of the group, start from 1.
    * @param y output projection of the vector to specific group.
    */
-  void ProjectToGroup(const arma::mat& v, const size_t groupId, arma::vec& y)
+  template<typename MatType>
+  void ProjectToGroup(const MatType& v, const size_t groupId, MatType& y)
   {
     arma::uvec& indList = groupIndicesList[groupId - 1];
     size_t dim = indList.n_elem;
-    y.set_size(dim);
+    y.set_size(dim, 1);
 
     for (size_t i = 0; i < dim; ++i)
       y(i) = v(indList(i));
@@ -160,14 +164,15 @@ class GroupLpBall
    * @param groupId optimal atom belongs to this group.
    * @param s output optimal atom.
    */
-  void OptimalFromGroup(const arma::mat& v, const size_t groupId, arma::mat& s)
+  template<typename MatType>
+  void OptimalFromGroup(const MatType& v, const size_t groupId, MatType& s)
   {
     // Project v to group.
-    arma::vec yk;
+    MatType yk;
     ProjectToGroup(v, groupId, yk);
 
     // Optimize in this group.
-    arma::vec sProj(yk.n_elem);
+    MatType sProj(yk.n_elem, 1);
     lpBallSolver.Optimize(yk, sProj);
 
     // Recover s to the original dimension.
@@ -190,7 +195,8 @@ class GroupLpBall
    * @param yk compute the q-norm of yk.
    * @param groupId group ID number.
    */
-  double DualNorm(const arma::vec& yk, const int groupId)
+  template<typename MatType>
+  typename MatType::elem_type DualNorm(const MatType& yk, const int groupId)
   {
     if (p == std::numeric_limits<double>::infinity())
     {
@@ -200,8 +206,8 @@ class GroupLpBall
     else if (p > 1.0)
     {
       // p norm, return q-norm
-      double q = 1.0 / (1.0 - 1.0/p);
-      return  arma::norm(yk, q);
+      double q = 1.0 / (1.0 - 1.0 / p);
+      return arma::norm(yk, q);
     }
     else if (p == 1.0)
     {
@@ -233,9 +239,6 @@ class GroupLpBall
   ConstrLpBallSolver lpBallSolver;
 };
 
-
 } // namespace ens
 
-
-
 #endif

include/ensmallen_bits/fw/frank_wolfe.hpp

@@ -114,13 +114,36 @@ class FrankWolfe
    *   void Gradient(const arma::mat& coordinates,
    *                 arma::mat& gradient);
    *
+   * @tparam FunctionType Type of function to be optimized.
+   * @tparam MatType Type of objective matrix.
+   * @tparam GradType Type of gradient matrix (default is MatType).
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to be optimized.
    * @param iterate Input with starting point, and will be modified to save
    *                the output optimial solution coordinates.
+   * @param callbacks Callback functions.
    * @return Objective value at the final solution.
    */
-  template<typename FunctionType>
-  double Optimize(FunctionType& function, arma::mat& iterate);
+  template<typename FunctionType, typename MatType, typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the linear constrained solver.
   const LinearConstrSolverType& LinearConstrSolver()

include/ensmallen_bits/fw/frank_wolfe_impl.hpp

@@ -39,34 +39,56 @@ FrankWolfe(const LinearConstrSolverType linearConstrSolver,
 template<
     typename LinearConstrSolverType,
     typename UpdateRuleType>
-template<typename FunctionType>
-double FrankWolfe<LinearConstrSolverType, UpdateRuleType>::
-Optimize(FunctionType& function, arma::mat& iterate)
+template<typename FunctionType, typename MatType, typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+FrankWolfe<LinearConstrSolverType, UpdateRuleType>::Optimize(
+  FunctionType& function,
+  MatType& iterateIn,
+  CallbackTypes&&... callbacks)
 {
-  typedef Function<FunctionType> FullFunctionType;
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
   FullFunctionType& f = static_cast<FullFunctionType&>(function);
 
   // Make sure we have all necessary functions.
-  traits::CheckFunctionTypeAPI<FullFunctionType>();
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
 
   // To keep track of the function value.
-  double currentObjective = DBL_MAX;
+  ElemType currentObjective = std::numeric_limits<ElemType>::max();
 
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  arma::mat s(iterate.n_rows, iterate.n_cols);
-  arma::mat iterateNew(iterate.n_rows, iterate.n_cols);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+  BaseMatType s(iterate.n_rows, iterate.n_cols);
+  BaseMatType iterateNew(iterate.n_rows, iterate.n_cols);
   double gap = 0;
 
-  for (size_t i = 1; i != maxIterations; ++i)
+  // Controls early termination of the optimization process.
+  bool terminate = false;
+
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t i = 1; i != maxIterations && !terminate; ++i)
   {
     currentObjective = f.EvaluateWithGradient(iterate, gradient);
 
+    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        currentObjective, gradient, callbacks...);
+
     // Output current objective function.
     Info << "FrankWolfe::Optimize(): iteration " << i << ", objective "
         << currentObjective << "." << std::endl;
 
     // Solve linear constrained problem, solution saved in s.
-    linearConstrSolver.Optimize(gradient, s);
+    linearConstrSolver.Optimize(gradient, s, callbacks...);
 
     // Check duality gap for return condition.
     gap = std::fabs(dot(iterate - s, gradient));
@@ -74,18 +96,23 @@ Optimize(FunctionType& function, arma::mat& iterate)
     {
       Info << "FrankWolfe::Optimize(): minimized within tolerance "
           << tolerance << "; " << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
       return currentObjective;
     }
 
-
     // Update solution, save in iterateNew.
-    updateRule.Update(f, iterate, s, iterateNew, i);
+    updateRule.template Update<FunctionType, BaseMatType, BaseGradType>(f,
+        iterate, s, iterateNew, i);
 
     iterate = std::move(iterateNew);
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
   }
 
   Info << "FrankWolfe::Optimize(): maximum iterations (" << maxIterations
       << ") reached; " << "terminating optimization." << std::endl;
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return currentObjective;
 } // Optimize()
 

include/ensmallen_bits/fw/line_search/line_search.hpp

@@ -44,8 +44,12 @@ class LineSearch
    *           coordinate of the optimal solution.
    * @return Minimum solution function value.
    */
-  template<typename FunctionType>
-  double Optimize(FunctionType& function, const arma::mat& x1, arma::mat& x2);
+  template<typename FunctionType,
+           typename MatType,
+           typename GradType = MatType>
+  typename MatType::elem_type Optimize(FunctionType& function,
+                                       const MatType& x1,
+                                       MatType& x2);
 
   //! Get the maximum number of iterations (0 indicates no limit).
   size_t MaxIterations() const { return maxIterations; }
@@ -74,11 +78,11 @@ class LineSearch
    *
    * @return Derivative of function(x0 + gamma * deltaX) with respect to gamma.
    */
-  template<typename FunctionType>
-  double Derivative(FunctionType& function,
-                    const arma::mat& x0,
-                    const arma::mat& deltaX,
-                    const double gamma);
+  template<typename FunctionType, typename MatType, typename GradType>
+  typename MatType::elem_type Derivative(FunctionType& function,
+                                         const MatType& x0,
+                                         const MatType& deltaX,
+                                         const double gamma);
 };  // class LineSearch
 } // namespace ens
 

include/ensmallen_bits/fw/line_search/line_search_impl.hpp

@@ -19,24 +19,28 @@
 
 namespace ens {
 
-template<typename FunctionType>
-double LineSearch::Optimize(FunctionType& function,
-                            const arma::mat& x1,
-                            arma::mat& x2)
+template<typename FunctionType, typename MatType, typename GradType>
+typename MatType::elem_type LineSearch::Optimize(FunctionType& function,
+                                                 const MatType& x1,
+                                                 MatType& x2)
 {
-  typedef Function<FunctionType> FullFunctionType;
+  typedef typename MatType::elem_type ElemType;
+
+  typedef Function<FunctionType, MatType, GradType> FullFunctionType;
   FullFunctionType& f = static_cast<FullFunctionType&>(function);
 
   // Check that we have all the functions we will need.
-  traits::CheckFunctionTypeAPI<FullFunctionType>();
+  traits::CheckFunctionTypeAPI<FullFunctionType, MatType, GradType>();
 
   // Set up the search line, that is,
   // find the zero of der(gamma) = Derivative(gamma).
-  arma::mat deltaX = x2 - x1;
-  double gamma = 0;
-  double derivative = Derivative(f, x1, deltaX, 0);
-  double derivativeNew = Derivative(f, x1, deltaX, 1);
-  double secant = derivativeNew - derivative;
+  MatType deltaX = x2 - x1;
+  ElemType gamma = 0;
+  ElemType derivative = Derivative<FunctionType, MatType, GradType>(f, x1,
+      deltaX, 0);
+  ElemType derivativeNew = Derivative<FunctionType, MatType, GradType>(f, x1,
+      deltaX, 1);
+  ElemType secant = derivativeNew - derivative;
 
   if (derivative >= 0.0) // Optimal solution at left endpoint.
   {
@@ -65,12 +69,13 @@ double LineSearch::Optimize(FunctionType& function,
     }
 
     // Solve new gamma.
-    double gammaNew = gamma - derivative / secant;
-    gammaNew = std::max(gammaNew, 0.0);
-    gammaNew = std::min(gammaNew, 1.0);
+    ElemType gammaNew = gamma - derivative / secant;
+    gammaNew = std::max(gammaNew, ElemType(0.0));
+    gammaNew = std::min(gammaNew, ElemType(1.0));
 
-    // Update secant, gamma and derivative
-    derivativeNew = Derivative(function, x1, deltaX, gammaNew);
+    // Update secant, gamma and derivative.
+    derivativeNew = Derivative<FunctionType, MatType, GradType>(function, x1,
+        deltaX, gammaNew);
     secant = (derivativeNew - derivative) / (gammaNew - gamma);
     gamma = gammaNew;
     derivative = derivativeNew;
@@ -93,17 +98,17 @@ double LineSearch::Optimize(FunctionType& function,
 
 
 //! Derivative of the function along the search line.
-template<typename FunctionType>
-double LineSearch::Derivative(FunctionType& function,
-                              const arma::mat& x0,
-                              const arma::mat& deltaX,
-                              const double gamma)
+template<typename FunctionType, typename MatType, typename GradType>
+typename MatType::elem_type LineSearch::Derivative(FunctionType& function,
+                                                   const MatType& x0,
+                                                   const MatType& deltaX,
+                                                   const double gamma)
 {
-  arma::mat gradient(x0.n_rows, x0.n_cols);
+  GradType gradient(x0.n_rows, x0.n_cols);
   function.Gradient(x0 + gamma * deltaX, gradient);
   return arma::dot(gradient, deltaX);
 }
 
-
 } // namespace ens
+
 #endif

include/ensmallen_bits/fw/proximal/proximal.hpp

@@ -35,7 +35,8 @@ class Proximal
    *          also saved in v.
    * @param tau Norm of l1 ball.
    */
-  static void ProjectToL1Ball(arma::vec& v, double tau);
+  template<typename MatType>
+  static void ProjectToL1Ball(MatType& v, double tau);
 
   /**
    * Project the vector onto the l0 ball with norm tau. That is, we try to
@@ -48,7 +49,8 @@ class Proximal
    *          also saved in v.
    * @param tau Norm of l0 ball.
    */
-  static void ProjectToL0Ball(arma::vec& v, int tau);
+  template<typename MatType>
+  static void ProjectToL0Ball(MatType& v, int tau);
 };  // class Proximal
 
 } // namespace ens

include/ensmallen_bits/fw/proximal/proximal_impl.hpp

@@ -32,30 +32,31 @@ namespace ens {
  *
  * This is just a soft thresholding.
  */
-inline void Proximal::ProjectToL1Ball(arma::vec& v, double tau)
+template<typename MatType>
+inline void Proximal::ProjectToL1Ball(MatType& v, double tau)
 {
-  arma::vec simplexSol = arma::abs(v);
+  MatType simplexSol = arma::abs(v);
 
   // Already with L1 norm <= tau.
   if (arma::accu(simplexSol) <= tau)
     return;
 
   simplexSol = arma::sort(simplexSol, "descend");
-  arma::vec simplexSum = arma::cumsum(simplexSol);
+  MatType simplexSum = arma::cumsum(simplexSol);
 
   double nu = 0;
-  size_t rho;
+  size_t rho = 0;
   for (size_t j = 1; j <= simplexSol.n_rows; j++)
   {
     rho = simplexSol.n_rows - j;
-    nu = simplexSol(rho) - (simplexSum(rho) - tau)/(rho + 1);
+    nu = simplexSol(rho) - (simplexSum(rho) - tau) / (rho + 1);
     if (nu > 0)
       break;
   }
-  double theta = (simplexSum(rho) - tau)/rho;
+  double theta = (simplexSum(rho) - tau) / rho;
 
   // Threshold on absolute value of v with theta.
-  for (arma::uword j = 0; j< simplexSol.n_rows; j++)
+  for (arma::uword j = 0; j < simplexSol.n_rows; j++)
   {
     if (v(j) >= 0.0)
       v(j) = std::max(v(j) - theta, 0.0);
@@ -68,7 +69,8 @@ inline void Proximal::ProjectToL1Ball(arma::vec& v, double tau)
  * Approximate the vector v with a tau-sparse vector.
  * This is a hard-thresholding.
  */
-inline void Proximal::ProjectToL0Ball(arma::vec& v, int tau)
+template<typename MatType>
+inline void Proximal::ProjectToL0Ball(MatType& v, int tau)
 {
   arma::uvec indices = arma::sort_index(arma::abs(v));
   arma::uword numberToKill = v.n_elem - tau;

include/ensmallen_bits/fw/update_classic.hpp

@@ -36,20 +36,20 @@ class UpdateClassic
    *
    * \f$ x_{k+1} = (1-\gamma)x_k + \gamma s \f$, where \f$ \gamma = 2/(k+2) \f$
    *
-   * @param function function to be optimized, not used in this update rule.
-   * @param oldCoords previous solution coords.
-   * @param s current linear_constr_solution result.
-   * @param newCoords output new solution coords.
-   * @param numIter current iteration number
+   * @param function Function to be optimized, not used in this update rule.
+   * @param oldCoords Previous solution coords.
+   * @param s Current linear_constr_solution result.
+   * @param newCoords Output new solution coords.
+   * @param numIter Current iteration number.
    */
-  template<typename FunctionType>
+  template<typename FunctionType, typename MatType, typename GradType>
   void Update(FunctionType& /* function */,
-              const arma::mat& oldCoords,
-              const arma::mat& s,
-              arma::mat& newCoords,
+              const MatType& oldCoords,
+              const MatType& s,
+              MatType& newCoords,
               const size_t numIter)
   {
-    double gamma = 2.0 / (numIter + 2.0);
+    typename MatType::elem_type gamma = 2.0 / (numIter + 2.0);
     newCoords = (1.0 - gamma) * oldCoords + gamma * s;
   }
 };

include/ensmallen_bits/fw/update_full_correction.hpp

@@ -49,31 +49,36 @@ class UpdateFullCorrection
    * Update rule for FrankWolfe, recalculate the coefficents of of current
    * atoms, while satisfying the norm constraint.
    *
-   * @param function function to be optimized.
-   * @param oldCoords previous solution coords.
-   * @param s current linear_constr_solution result.
-   * @param newCoords new output solution coords.
-   * @param numIter current iteration number.
+   * FuncSqType is an ignored type to match the requirements of the class.
+   *
+   * @param function Function to be optimized.
+   * @param oldCoords Previous solution coords.
+   * @param s Current linear_constr_solution result.
+   * @param newCoords New output solution coords.
+   * @param numIter Current iteration number.
    */
+  template<typename FuncSqType, typename MatType, typename GradType>
   void Update(FuncSq& function,
-              const arma::mat& oldCoords,
-              const arma::mat& s,
-              arma::mat& newCoords,
+              const MatType& oldCoords,
+              const MatType& s,
+              MatType& newCoords,
               const size_t /* numIter */)
   {
     // Line search, with explicit solution here.
-    arma::mat v = tau * s - oldCoords;
-    arma::mat b = function.Vectorb();
-    arma::mat A = function.MatrixA();
-    double gamma = arma::dot(b - A * oldCoords, A * v);
+    MatType v = tau * s - oldCoords;
+    MatType b = function.Vectorb();
+    MatType A = function.MatrixA();
+    typename MatType::elem_type gamma = arma::dot(b - A * oldCoords, A * v);
     gamma = gamma / std::pow(arma::norm(A * v, "fro"), 2);
     gamma = std::min(gamma, 1.0);
     atoms.CurrentCoeffs() = (1.0 - gamma) * atoms.CurrentCoeffs();
-    atoms.AddAtom(s, function, gamma * tau);
+    atoms.AddAtom(arma::mat(s), function, gamma * tau);
 
     // Projected gradient method for enhancement.
     atoms.ProjectedGradientEnhancement(function, tau, stepSize);
-    atoms.RecoverVector(newCoords);
+    arma::mat tmp;
+    atoms.RecoverVector(tmp);
+    newCoords = arma::conv_to<MatType>::from(tmp);
   }
 
  private:

include/ensmallen_bits/fw/update_linesearch.hpp

@@ -40,7 +40,6 @@ class UpdateLineSearch
       tolerance(tolerance), maxIterations(maxIterations)
   {/* Do nothing */}
 
-
   /**
    * Update rule for FrankWolfe, optimize with line search using secant method.
    *
@@ -64,18 +63,19 @@ class UpdateLineSearch
    * @param newCoords output new solution coords.
    * @param numIter current iteration number, not used here.
    */
-  template<typename FunctionType>
+  template<typename FunctionType, typename MatType, typename GradType>
   void Update(FunctionType& function,
-              const arma::mat& oldCoords,
-              const arma::mat& s,
-              arma::mat& newCoords,
+              const MatType& oldCoords,
+              const MatType& s,
+              MatType& newCoords,
               const size_t /* numIter */)
 
   {
     LineSearch solver(maxIterations, tolerance);
 
     newCoords = s;
-    solver.Optimize(function, oldCoords, newCoords);
+    solver.Optimize<FunctionType, MatType, GradType>(function, oldCoords,
+        newCoords);
   }
 
   //! Get the tolerance for termination.

include/ensmallen_bits/fw/update_span.hpp

@@ -45,22 +45,25 @@ class UpdateSpan
    * @param newCoords output new solution coords.
    * @param numIter current iteration number.
    */
+  template<typename FuncSqType, typename MatType, typename GradType>
   void Update(FuncSq& function,
-              const arma::mat& oldCoords,
-              const arma::mat& s,
-              arma::mat& newCoords,
+              const MatType& oldCoords,
+              const MatType& s,
+              MatType& newCoords,
               const size_t /* numIter */)
   {
     // Add new atom into soluton space.
-    atoms.AddAtom(s, function);
+    atoms.AddAtom(arma::mat(s), function);
 
     // Reoptimize the solution in the current space.
     arma::vec b = function.Vectorb();
-    atoms.CurrentCoeffs() = solve(function.MatrixA() * atoms.CurrentAtoms(), b);
+    atoms.CurrentCoeffs() = solve(function.MatrixA() * atoms.CurrentAtoms(), b, arma::solve_opts::fast);
 
     // x has coords of only the current atoms, recover the solution
     // to the original size.
-    atoms.RecoverVector(newCoords);
+    arma::mat tmp;
+    atoms.RecoverVector(tmp);
+    newCoords = arma::conv_to<MatType>::from(tmp);
 
     // Prune the support.
     if (isPrune)
@@ -68,7 +71,8 @@ class UpdateSpan
       double oldF = function.Evaluate(oldCoords);
       double F = 0.25 * oldF + 0.75 * function.Evaluate(newCoords);
       atoms.PruneSupport(F, function);
-      atoms.RecoverVector(newCoords);
+      atoms.RecoverVector(tmp);
+      newCoords = arma::conv_to<MatType>::from(tmp);
     }
   }
 

include/ensmallen_bits/gradient_descent/gradient_descent.hpp

@@ -65,12 +65,36 @@ class GradientDescent
    * the final objective value is returned.
    *
    * @tparam FunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename FunctionType>
-  double Optimize(FunctionType& function, arma::mat& iterate);
+  template<typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   /**
    * Assert all dimensions are numeric and optimize the given function using
@@ -82,19 +106,44 @@ class GradientDescent
    * tuning module.
    *
    * @tparam FunctionType Type of the function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
    * @param categoricalDimensions A vector of dimension information.  If a value
    *     is true, then that dimension is a categorical dimension.
    * @param numCategories Number of categories in each categorical dimension.
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename FunctionType>
-  double Optimize(
-       FunctionType& function,
-       arma::mat& iterate,
-       const std::vector<bool>& categoricalDimensions,
-       const arma::Row<size_t>& numCategories);
+  template<typename FunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(FunctionType& function,
+           MatType& iterate,
+           const std::vector<bool>& categoricalDimensions,
+           const arma::Row<size_t>& numCategories,
+           CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename FunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(
+      FunctionType& function,
+      MatType& iterate,
+      const std::vector<bool>& categoricalDimensions,
+      const arma::Row<size_t>& numCategories,
+      CallbackTypes&&... callbacks)
+  {
+    return Optimize<FunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate, categoricalDimensions,
+        numCategories, std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the step size.
   double StepSize() const { return stepSize; }

include/ensmallen_bits/gradient_descent/gradient_descent_impl.hpp

@@ -30,27 +30,50 @@ inline GradientDescent::GradientDescent(
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
-template<typename FunctionType>
-double GradientDescent::Optimize(
-    FunctionType& function, arma::mat& iterate)
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+GradientDescent::Optimize(FunctionType& function,
+                          MatType& iterateIn,
+                          CallbackTypes&&... callbacks)
 {
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
   // Use the Function<> wrapper type to provide additional functionality.
-  typedef Function<FunctionType> FullFunctionType;
+  typedef Function<FunctionType, BaseMatType, BaseGradType> FullFunctionType;
   FullFunctionType& f(static_cast<FullFunctionType&>(function));
 
   // Make sure we have the methods that we need.
-  traits::CheckFunctionTypeAPI<FullFunctionType>();
+  traits::CheckFunctionTypeAPI<FullFunctionType, BaseMatType, BaseGradType>();
+  RequireFloatingPointType<BaseMatType>();
+  RequireFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
 
   // To keep track of where we are and how things are going.
-  double overallObjective = std::numeric_limits<double>::max();
-  double lastObjective = std::numeric_limits<double>::max();
+  ElemType overallObjective = std::numeric_limits<ElemType>::max();
+  ElemType lastObjective = std::numeric_limits<ElemType>::max();
+
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
   // Now iterate!
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  for (size_t i = 1; i != maxIterations; ++i)
+  terminate |= Callback::BeginOptimization(*this, f, iterate, callbacks...);
+  for (size_t i = 1; i != maxIterations && !terminate; ++i)
   {
     overallObjective = f.EvaluateWithGradient(iterate, gradient);
 
+    terminate |= Callback::EvaluateWithGradient(*this, f, iterate,
+        overallObjective, gradient, callbacks...);
+
     // Output current objective function.
     Info << "Gradient Descent: iteration " << i << ", objective "
         << overallObjective << "." << std::endl;
@@ -60,6 +83,8 @@ double GradientDescent::Optimize(
       Warn << "Gradient Descent: converged to " << overallObjective
           << "; terminating" << " with failure.  Try a smaller step size?"
           << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -67,6 +92,8 @@ double GradientDescent::Optimize(
     {
       Info << "Gradient Descent: minimized within tolerance "
           << tolerance << "; " << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, f, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -75,19 +102,28 @@ double GradientDescent::Optimize(
 
     // And update the iterate.
     iterate -= stepSize * gradient;
+    terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
   }
 
   Info << "Gradient Descent: maximum iterations (" << maxIterations
       << ") reached; " << "terminating optimization." << std::endl;
+
+  Callback::EndOptimization(*this, f, iterate, callbacks...);
   return overallObjective;
 }
 
-template<typename FunctionType>
-double GradientDescent::Optimize(
+template<typename FunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+GradientDescent::Optimize(
     FunctionType& function,
-    arma::mat& iterate,
+    MatType& iterate,
     const std::vector<bool>& categoricalDimensions,
-    const arma::Row<size_t>& numCategories)
+    const arma::Row<size_t>& numCategories,
+    CallbackTypes&&... callbacks)
 {
   if (categoricalDimensions.size() != iterate.n_rows)
   {
@@ -118,7 +154,7 @@ double GradientDescent::Optimize(
     }
   }
 
-  return Optimize(function, iterate);
+  return Optimize(function, iterate, callbacks...);
 }
 
 } // namespace ens

include/ensmallen_bits/grid_search/grid_search.hpp

@@ -30,6 +30,8 @@ class GridSearch
    * possible combinations of values for the parameters specified in
    * datasetInfo.
    *
+   * @tparam FunctionType Type of function to optimize.
+   * @tparam MatType Type of matrix to optimize with.
    * @param function Function to optimize.
    * @param bestParameters Variable for storing results.
    * @param categoricalDimensions Set of dimension types.  If a value is true,
@@ -37,10 +39,10 @@ class GridSearch
    * @param numCategories Number of categories in each categorical dimension.
    * @return Objective value of the final point.
    */
-  template<typename FunctionType>
-  double Optimize(
+  template<typename FunctionType, typename MatType>
+  typename MatType::elem_type Optimize(
       FunctionType& function,
-      arma::mat& bestParameters,
+      MatType& bestParameters,
       const std::vector<bool>& categoricalDimensions,
       const arma::Row<size_t>& numCategories);
 
@@ -52,12 +54,12 @@ class GridSearch
    * (parameters) are specified in the first i rows of the currentParameters
    * argument.
    */
-  template<typename FunctionType>
+  template<typename FunctionType, typename MatType>
   void Optimize(
       FunctionType& function,
-      double& bestObjective,
-      arma::mat& bestParameters,
-      arma::vec& currentParameters,
+      typename MatType::elem_type& bestObjective,
+      MatType& bestParameters,
+      MatType& currentParameters,
       const std::vector<bool>& categoricalDimensions,
       const arma::Row<size_t>& numCategories,
       size_t i);

include/ensmallen_bits/grid_search/grid_search_impl.hpp

@@ -17,10 +17,10 @@
 
 namespace ens {
 
-template<typename FunctionType>
-double GridSearch::Optimize(
+template<typename FunctionType, typename MatType>
+typename MatType::elem_type GridSearch::Optimize(
     FunctionType& function,
-    arma::mat& bestParameters,
+    MatType& bestParameters,
     const std::vector<bool>& categoricalDimensions,
     const arma::Row<size_t>& numCategories)
 {
@@ -35,9 +35,12 @@ double GridSearch::Optimize(
     }
   }
 
-  double bestObjective = std::numeric_limits<double>::max();
-  bestParameters = arma::mat(categoricalDimensions.size(), 1);
-  arma::vec currentParameters = arma::vec(categoricalDimensions.size());
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+
+  ElemType bestObjective = std::numeric_limits<ElemType>::max();
+  bestParameters.set_size(categoricalDimensions.size(), 1);
+  MatType currentParameters(categoricalDimensions.size(), 1);
 
   /* Initialize best parameters for the case (very unlikely though) when no set
    * of parameters gives an objective value better than
@@ -51,18 +54,23 @@ double GridSearch::Optimize(
   return bestObjective;
 }
 
-template<typename FunctionType>
+template<typename FunctionType, typename MatType>
 void GridSearch::Optimize(
     FunctionType& function,
-    double& bestObjective,
-    arma::mat& bestParameters,
-    arma::vec& currentParameters,
+    typename MatType::elem_type& bestObjective,
+    MatType& bestParameters,
+    MatType& currentParameters,
     const std::vector<bool>& categoricalDimensions,
     const arma::Row<size_t>& numCategories,
     size_t i)
 {
-  // Make sure we have the methods that we need.
-  traits::CheckNonDifferentiableFunctionTypeAPI<FunctionType>();
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+
+  // Make sure we have the methods that we need.  No restrictions on the matrix
+  // type are needed.
+  traits::CheckArbitraryFunctionTypeAPI<FunctionType, BaseMatType>();
 
   if (i < categoricalDimensions.size())
   {
@@ -75,7 +83,7 @@ void GridSearch::Optimize(
   }
   else
   {
-    double objective = function.Evaluate(currentParameters);
+    ElemType objective = function.Evaluate((BaseMatType&) currentParameters);
     if (objective < bestObjective)
     {
       bestObjective = objective;

include/ensmallen_bits/iqn/iqn.hpp

@@ -74,13 +74,37 @@ class IQN
    * modified to store the finishing point of the algorithm, and the final
    * objective value is returned.
    *
-   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @tparam SeparableFunctionType Type of the function to be optimized.
+   * @tparam MatType Type of matrix to optimize with.
+   * @tparam GradType Type of matrix to use to represent function gradients.
+   * @tparam CallbackTypes Types of callback functions.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param callbacks Callback functions.
    * @return Objective value of the final point.
    */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename GradType,
+           typename... CallbackTypes>
+  typename std::enable_if<IsArmaType<GradType>::value,
+      typename MatType::elem_type>::type
+  Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks);
+
+  //! Forward the MatType as GradType.
+  template<typename SeparableFunctionType,
+           typename MatType,
+           typename... CallbackTypes>
+  typename MatType::elem_type Optimize(SeparableFunctionType& function,
+                                       MatType& iterate,
+                                       CallbackTypes&&... callbacks)
+  {
+    return Optimize<SeparableFunctionType, MatType, MatType,
+        CallbackTypes...>(function, iterate,
+        std::forward<CallbackTypes>(callbacks)...);
+  }
 
   //! Get the step size.
   double StepSize() const { return stepSize; }

include/ensmallen_bits/iqn/iqn_impl.hpp

@@ -32,10 +32,34 @@ inline IQN::IQN(const double stepSize,
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
-template<typename DecomposableFunctionType>
-double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+template<typename SeparableFunctionType,
+         typename MatType,
+         typename GradType,
+         typename... CallbackTypes>
+typename std::enable_if<IsArmaType<GradType>::value,
+typename MatType::elem_type>::type
+IQN::Optimize(SeparableFunctionType& functionIn,
+              MatType& iterateIn,
+              CallbackTypes&&... callbacks)
 {
-  traits::CheckDecomposableFunctionTypeAPI<DecomposableFunctionType>();
+  // Convenience typedefs.
+  typedef typename MatType::elem_type ElemType;
+  typedef typename MatTypeTraits<MatType>::BaseMatType BaseMatType;
+  typedef typename MatTypeTraits<GradType>::BaseMatType BaseGradType;
+
+  typedef Function<SeparableFunctionType, BaseMatType, BaseGradType>
+      FullFunctionType;
+  FullFunctionType& function(static_cast<FullFunctionType&>(functionIn));
+
+  // Make sure we have all the methods that we need.
+  traits::CheckSeparableFunctionTypeAPI<FullFunctionType, BaseMatType,
+      BaseGradType>();
+  RequireDenseFloatingPointType<BaseMatType>();
+  RequireDenseFloatingPointType<BaseGradType>();
+  RequireSameInternalTypes<BaseMatType, BaseGradType>();
+
+  traits::CheckSeparableFunctionTypeAPI<SeparableFunctionType,
+      BaseMatType, BaseGradType>();
 
   // Find the number of functions.
   const size_t numFunctions = function.NumFunctions();
@@ -43,42 +67,54 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
   if (numFunctions % batchSize != 0)
     ++numBatches; // Capture last few.
 
+  BaseMatType& iterate = (BaseMatType&) iterateIn;
+
   // To keep track of where we are and how things are going.
-  double overallObjective = 0;
+  ElemType overallObjective = 0;
 
-  arma::cube y(iterate.n_rows, iterate.n_cols, numBatches);
-  arma::cube t(iterate.n_elem, 1, numBatches);
-  arma::cube Q(iterate.n_elem, iterate.n_elem, numBatches);
-  arma::mat initialIterate = arma::randn(iterate.n_rows, iterate.n_cols);
-  arma::mat B = arma::eye(iterate.n_elem, iterate.n_elem);
+  // Controls early termination of the optimization process.
+  bool terminate = false;
 
-  arma::mat g = arma::zeros(iterate.n_rows, iterate.n_cols);
+  std::vector<BaseGradType> y(numBatches, BaseGradType(iterate.n_rows,
+      iterate.n_cols));
+  std::vector<BaseMatType> t(numBatches, BaseMatType(iterate.n_rows,
+      iterate.n_cols));
+  std::vector<BaseMatType> Q(numBatches, BaseMatType(iterate.n_elem,
+      iterate.n_elem));
+  BaseMatType initialIterate = arma::randn<arma::Mat<ElemType>>(iterate.n_rows,
+      iterate.n_cols);
+  BaseGradType B(iterate.n_elem, iterate.n_elem);
+  B.eye();
+
+  BaseGradType g(iterate.n_rows, iterate.n_cols);
+  g.zeros();
   for (size_t i = 0, f = 0; i < numFunctions; f++)
   {
     // Find the effective batch size (the last batch may be smaller).
     const size_t effectiveBatchSize = std::min(batchSize, numFunctions - i);
 
-    t.slice(f) = arma::mat(initialIterate.memptr(), iterate.n_elem,
-        1, false, false);
-    function.Gradient(initialIterate, i, y.slice(f), effectiveBatchSize);
+    // It would be nice to avoid this copy but it is difficult to be generic to
+    // any MatType and still do that.
+    t[f] = initialIterate;
+    function.Gradient(initialIterate, i, y[f], effectiveBatchSize);
 
-    Q.slice(f).eye();
-    g += y.slice(f);
-    y.slice(f) /= (double) effectiveBatchSize;
+    terminate |= Callback::Gradient(*this, function, initialIterate,
+        y[f], callbacks...);
+
+    Q[f].eye();
+    g += y[f];
+    y[f] /= (double) effectiveBatchSize;
 
     i += effectiveBatchSize;
   }
   g /= numFunctions;
 
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-  arma::mat u = t.slice(0);
+  BaseGradType gradient(iterate.n_rows, iterate.n_cols);
+  BaseMatType u = t[0];
 
-  // Convenience alias to avoid multiple use of arma::vectorise.
-  arma::mat iterateVec = arma::mat(iterate.memptr(), iterate.n_elem,
-      1, false, false);
-  arma::mat gVec = arma::mat(g.memptr(), iterate.n_elem, 1, false, false);
-
-  for (size_t i = 1; i != maxIterations; ++i)
+  terminate |= Callback::BeginOptimization(*this, function, iterate,
+      callbacks...);
+  for (size_t i = 1; i != maxIterations && !terminate; ++i)
   {
     for (size_t j = 0, f = 0; f < numFunctions; j++)
     {
@@ -89,36 +125,43 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions -
           it * batchSize);
 
-      if (arma::norm(iterateVec - t.slice(it)) > 0)
+      if (arma::norm(iterate - t[it]) > 0)
       {
         function.Gradient(iterate, it * batchSize, gradient,
             effectiveBatchSize);
         gradient /= effectiveBatchSize;
 
-        const arma::mat s = iterateVec - t.slice(it);
-        const arma::mat yy = arma::vectorise(gradient - y.slice(it));
+        terminate |= Callback::Gradient(*this, function, iterate, gradient,
+            callbacks...);
 
-        const arma::mat stochasticHessian = Q.slice(it) + yy * yy.t() /
-            arma::as_scalar(yy.t() * s) - Q.slice(it) * s * s.t() *
-            Q.slice(it) / arma::as_scalar(s.t() * Q.slice(it) * s);
+        const BaseMatType s = arma::vectorise(iterate - t[it]);
+        const BaseGradType yy = arma::vectorise(gradient - y[it]);
+
+        const BaseGradType stochasticHessian = Q[it] + yy * yy.t() /
+            arma::as_scalar(yy.t() * s) - Q[it] * s * s.t() *
+            Q[it] / arma::as_scalar(s.t() * Q[it] * s);
 
         // Update aggregate Hessian approximation.
-        B += (1.0 / numBatches) * (stochasticHessian - Q.slice(it));
+        B += (1.0 / numBatches) * (stochasticHessian - Q[it]);
 
         // Update aggregate Hessian-variable product.
-        u += (1.0 / numBatches) * (stochasticHessian * iterateVec -
-            Q.slice(it) * t.slice(it));
+        u += arma::reshape((1.0 / numBatches) * (stochasticHessian *
+            arma::vectorise(iterate) - Q[it] * arma::vectorise(t[it])),
+            u.n_rows, u.n_cols);;
 
         // Update aggregate gradient.
-        g += (1.0 / numBatches) * (gradient - y.slice(it));
+        g += (1.0 / numBatches) * (gradient - y[it]);
 
         // Update the function information tables.
-        Q.slice(it) = stochasticHessian;
-        y.slice(it) = gradient;
-        t.slice(it) = iterateVec;
+        Q[it] = std::move(stochasticHessian);
+        y[it] = std::move(gradient);
+        t[it] = iterate;
 
-        iterateVec = stepSize * B.i() * (u - gVec) + (1 - stepSize) *
-            iterateVec;
+        iterate = arma::reshape(stepSize * B.i() * (u.t() - arma::vectorise(g)),
+            iterate.n_rows, iterate.n_cols) + (1 - stepSize) * iterate;
+
+        terminate |= Callback::StepTaken(*this, function, iterate,
+            callbacks...);
       }
 
       f += effectiveBatchSize;
@@ -128,7 +171,12 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
     for (size_t f = 0; f < numFunctions; f += batchSize)
     {
       const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f);
-      overallObjective += function.Evaluate(iterate, f, effectiveBatchSize);
+      const ElemType objective = function.Evaluate(iterate, f,
+          effectiveBatchSize);
+      overallObjective += objective;
+
+      Callback::Evaluate(*this, function, iterate, objective,
+          callbacks...);
     }
     overallObjective /= numFunctions;
 
@@ -140,6 +188,8 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
     {
       Warn << "IQN: converged to " << overallObjective << "; terminating"
           << " with failure.  Try a smaller step size?" << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
 
@@ -147,6 +197,8 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
     {
       Info << "IQN: minimized within tolerance " << tolerance << "; "
           << "terminating optimization." << std::endl;
+
+      Callback::EndOptimization(*this, function, iterate, callbacks...);
       return overallObjective;
     }
   }
@@ -154,6 +206,7 @@ double IQN::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
   Info << "IQN: maximum iterations (" << maxIterations << ") reached; "
       << "terminating optimization." << std::endl;
 
+  Callback::EndOptimization(*this, function, iterate, callbacks...);
   return overallObjective;
 }