diff --git a/.appveyor.yml b/.appveyor.yml index 068f2e4c76..f0ffa39490 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -85,8 +85,8 @@ build_script: - cd C:\projects\mlpack && mkdir build && cd build - > cmake -G "%VSVER%" - -DBLAS_LIBRARY:FILEPATH=%BLAS_LIBRARY% - -DLAPACK_LIBRARY:FILEPATH=%BLAS_LIBRARY% + -DBLAS_LIBRARIES:FILEPATH=%BLAS_LIBRARY% + -DLAPACK_LIBRARIES:FILEPATH=%BLAS_LIBRARY% -DARMADILLO_INCLUDE_DIR="C:/projects/mlpack/armadillo-8.400.0/include" -DARMADILLO_LIBRARY:FILEPATH=%ARMADILLO_LIBRARY% -DBOOST_INCLUDEDIR:PATH=%BOOST_INCLUDE% diff --git a/.ci/ci.yaml b/.ci/ci.yaml index be1ca4b4c1..e0654b49e1 100644 --- a/.ci/ci.yaml +++ b/.ci/ci.yaml @@ -1,3 +1,12 @@ +trigger: + branches: + include: + - '*' +pr: + branches: + include: + - '*' + jobs: - job: Linux timeoutInMinutes: 360 @@ -28,7 +37,7 @@ jobs: - job: macOS timeoutInMinutes: 360 pool: - vmImage: macOS-10.13 + vmImage: macOS-10.14 strategy: matrix: Plain: @@ -48,23 +57,6 @@ jobs: steps: - template: macos-steps.yaml -- job: WindowsVS14 - timeoutInMinutes: 360 - displayName: Windows VS14 - pool: - vmImage: vs2015-win2012r2 - strategy: - matrix: - Plain: - CMakeArgs: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF' - CMakeGenerator: '-G "Visual Studio 14 2015 Win64"' - MSBuildVersion: '14.0' - ArchiveNoLibs: 'mlpack-windows-vs14-no-libs.zip' - ArchiveLibs: 'mlpack-windows-vs14.zip' - ArchiveTests: 'mlpack_test-vs14.xml' - steps: - - template: windows-steps.yaml - - job: WindowsVS15 timeoutInMinutes: 360 displayName: Windows VS15 diff --git a/.ci/macos-steps.yaml b/.ci/macos-steps.yaml index bdc11fe3f7..968abd018f 100644 --- a/.ci/macos-steps.yaml +++ b/.ci/macos-steps.yaml @@ -14,8 +14,7 @@ steps: set -e sudo xcode-select --switch /Applications/Xcode_10.1.app/Contents/Developer unset BOOST_ROOT - pip install cython numpy pandas zipp - brew update + pip install cython numpy pandas zipp configparser brew install openblas armadillo boost if [ "a$(julia.version)" != "a" ]; then diff --git a/.ci/windows-steps.yaml b/.ci/windows-steps.yaml index 4d8b9e7923..26cd2c0ff4 100644 --- a/.ci/windows-steps.yaml +++ b/.ci/windows-steps.yaml @@ -28,7 +28,7 @@ steps: # Configure armadillo - bash: | git clone --depth 1 https://github.com/mlpack/jenkins-conf.git conf - + curl -O http://masterblaster.mlpack.org:5005/armadillo-8.400.0.tar.gz -o armadillo-8.400.0.tar.gz tar -xzvf armadillo-8.400.0.tar.gz @@ -60,12 +60,13 @@ steps: cmake $(CMakeGenerator) ` $(CMakeArgs) ` - -DBLAS_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a ` - -DLAPACK_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a ` + -DBLAS_LIBRARIES:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a ` + -DLAPACK_LIBRARIES:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a ` -DARMADILLO_INCLUDE_DIR="..\armadillo-8.400.0\include" ` -DARMADILLO_LIBRARY="..\armadillo-8.400.0\Release\armadillo.lib" ` -DBOOST_INCLUDEDIR=$(Agent.ToolsDirectory)\boost.1.60.0.0\lib\native\include ` -DBOOST_LIBRARYDIR=$(Agent.ToolsDirectory)\boost_libs ` + -DBUILD_JULIA_BINDINGS=OFF ` -DCMAKE_BUILD_TYPE=Release .. displayName: 'Configure mlpack' diff --git a/CMake/ARMA_FindACML.cmake b/CMake/ARMA_FindACML.cmake deleted file mode 100644 index 42561cafda..0000000000 --- a/CMake/ARMA_FindACML.cmake +++ /dev/null @@ -1,37 +0,0 @@ -# - Find AMD's ACML library (no includes) which provides optimised BLAS and LAPACK functions -# This module defines -# ACML_LIBRARIES, the libraries needed to use ACML. -# ACML_FOUND, If false, do not try to use ACML. -# also defined, but not for general use are -# ACML_LIBRARY, where to find the ACML library. - -set(ACML_NAMES ${ACML_NAMES} acml) -find_library(ACML_LIBRARY - NAMES ${ACML_NAMES} - PATHS /usr/lib64 /usr/lib /usr/*/lib64 /usr/*/lib /usr/*/gfortran64/lib/ /usr/*/gfortran32/lib/ /usr/local/lib64 /usr/local/lib /opt/lib64 /opt/lib /opt/*/lib64 /opt/*/lib /opt/*/gfortran64/lib/ /opt/*/gfortran32/lib/ - ) - -if (ACML_LIBRARY) - set(ACML_LIBRARIES ${ACML_LIBRARY}) - set(ACML_FOUND "YES") -else () - set(ACML_FOUND "NO") -endif () - - -if (ACML_FOUND) - if (NOT ACML_FIND_QUIETLY) - message(STATUS "Found the ACML library: ${ACML_LIBRARIES}") - endif () -else () - if (ACML_FIND_REQUIRED) - message(FATAL_ERROR "Could not find the ACML library") - endif () -endif () - -# Deprecated declarations. -get_filename_component (NATIVE_ACML_LIB_PATH ${ACML_LIBRARY} PATH) - -mark_as_advanced( - ACML_LIBRARY - ) diff --git a/CMake/ARMA_FindACMLMP.cmake b/CMake/ARMA_FindACMLMP.cmake deleted file mode 100644 index 47a192ced6..0000000000 --- a/CMake/ARMA_FindACMLMP.cmake +++ /dev/null @@ -1,37 +0,0 @@ -# - Find AMD's ACMLMP library (no includes) which provides optimised and parallelised BLAS and LAPACK functions -# This module defines -# ACMLMP_LIBRARIES, the libraries needed to use ACMLMP. -# ACMLMP_FOUND, If false, do not try to use ACMLMP. -# also defined, but not for general use are -# ACMLMP_LIBRARY, where to find the ACMLMP library. - -set(ACMLMP_NAMES ${ACMLMP_NAMES} acml_mp) -find_library(ACMLMP_LIBRARY - NAMES ${ACMLMP_NAMES} - PATHS /usr/lib64 /usr/lib /usr/*/lib64 /usr/*/lib /usr/*/gfortran64_mp/lib/ /usr/*/gfortran32_mp/lib/ /usr/local/lib64 /usr/local/lib /opt/lib64 /opt/lib /opt/*/lib64 /opt/*/lib /opt/*/gfortran64_mp/lib/ /opt/*/gfortran32_mp/lib/ - ) - -if (ACMLMP_LIBRARY) - set(ACMLMP_LIBRARIES ${ACMLMP_LIBRARY}) - set(ACMLMP_FOUND "YES") -else () - set(ACMLMP_FOUND "NO") -endif () - - -if (ACMLMP_FOUND) - if (NOT ACMLMP_FIND_QUIETLY) - message(STATUS "Found the ACMLMP library: ${ACMLMP_LIBRARIES}") - endif () -else () - if (ACMLMP_FIND_REQUIRED) - message(FATAL_ERROR "Could not find the ACMLMP library") - endif () -endif () - -# Deprecated declarations. -get_filename_component (NATIVE_ACMLMP_LIB_PATH ${ACMLMP_LIBRARY} PATH) - -mark_as_advanced( - ACMLMP_LIBRARY - ) diff --git a/CMake/ARMA_FindARPACK.cmake b/CMake/ARMA_FindARPACK.cmake deleted file mode 100644 index ff1ee22797..0000000000 --- a/CMake/ARMA_FindARPACK.cmake +++ /dev/null @@ -1,39 +0,0 @@ -# - Try to find ARPACK -# Once done this will define -# -# ARPACK_FOUND - system has ARPACK -# ARPACK_LIBRARY - Link this to use ARPACK - - -find_library(ARPACK_LIBRARY - NAMES arpack - PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib - ) - - -if (ARPACK_LIBRARY) - set(ARPACK_FOUND YES) -else () - # Search for PARPACK. - find_library(ARPACK_LIBRARY - NAMES parpack - PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib - ) - - if (ARPACK_LIBRARY) - set(ARPACK_FOUND YES) - else () - set(ARPACK_FOUND NO) - endif () -endif () - - -if (ARPACK_FOUND) - if (NOT ARPACK_FIND_QUIETLY) - message(STATUS "Found an ARPACK library: ${ARPACK_LIBRARY}") - endif () -else () - if (ARPACK_FIND_REQUIRED) - message(FATAL_ERROR "Could not find an ARPACK library") - endif () -endif () diff --git a/CMake/ARMA_FindBLAS.cmake b/CMake/ARMA_FindBLAS.cmake deleted file mode 100644 index ff37b4fc79..0000000000 --- a/CMake/ARMA_FindBLAS.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# - Find a BLAS library (no includes) -# This module defines -# BLAS_LIBRARIES, the libraries needed to use BLAS. -# BLAS_FOUND, If false, do not try to use BLAS. -# also defined, but not for general use are -# BLAS_LIBRARY, where to find the BLAS library. - -set(BLAS_NAMES ${BLAS_NAMES} blas) - -# Find the ATLAS version preferentially. -find_library(BLAS_LIBRARY - NAMES ${BLAS_NAMES} - PATHS /usr/lib64/atlas /usr/lib/atlas /usr/local/lib64/atlas /usr/local/lib/atlas - NO_DEFAULT_PATH) - -find_library(BLAS_LIBRARY - NAMES ${BLAS_NAMES} - PATHS /usr/lib64/atlas /usr/lib/atlas /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib - ) - -if (BLAS_LIBRARY) - set(BLAS_LIBRARIES ${BLAS_LIBRARY}) - set(BLAS_FOUND "YES") -else () - set(BLAS_FOUND "NO") -endif () - - -if (BLAS_FOUND) - if (NOT BLAS_FIND_QUIETLY) - message(STATUS "Found BLAS: ${BLAS_LIBRARIES}") - endif () -else () - if (BLAS_FIND_REQUIRED) - message(FATAL_ERROR "Could not find BLAS") - endif () -endif () - -# Deprecated declarations. -get_filename_component (NATIVE_BLAS_LIB_PATH ${BLAS_LIBRARY} PATH) - -mark_as_advanced( - BLAS_LIBRARY - ) diff --git a/CMake/ARMA_FindCBLAS.cmake b/CMake/ARMA_FindCBLAS.cmake deleted file mode 100644 index da84c246b8..0000000000 --- a/CMake/ARMA_FindCBLAS.cmake +++ /dev/null @@ -1,47 +0,0 @@ -# - Find CBLAS (includes and library) -# This module defines -# CBLAS_INCLUDE_DIR -# CBLAS_LIBRARIES -# CBLAS_FOUND -# also defined, but not for general use are -# CBLAS_LIBRARY, where to find the library. - -find_path(CBLAS_INCLUDE_DIR cblas.h -/usr/include/atlas/ -/usr/local/include/atlas/ -/usr/include/ -/usr/local/include/ -) - -set(CBLAS_NAMES ${CBLAS_NAMES} cblas) -find_library(CBLAS_LIBRARY - NAMES ${CBLAS_NAMES} - PATHS /usr/lib64/atlas-sse3 /usr/lib64/atlas /usr/lib64 /usr/local/lib64/atlas /usr/local/lib64 /usr/lib/atlas-sse3 /usr/lib/atlas-sse2 /usr/lib/atlas-sse /usr/lib/atlas-3dnow /usr/lib/atlas /usr/lib /usr/local/lib/atlas /usr/local/lib - ) - -if (CBLAS_LIBRARY AND CBLAS_INCLUDE_DIR) - set(CBLAS_LIBRARIES ${CBLAS_LIBRARY}) - set(CBLAS_FOUND "YES") -else () - set(CBLAS_FOUND "NO") -endif () - - -if (CBLAS_FOUND) - if (NOT CBLAS_FIND_QUIETLY) - message(STATUS "Found a CBLAS library: ${CBLAS_LIBRARIES}") - endif () -else () - if (CBLAS_FIND_REQUIRED) - message(FATAL_ERROR "Could not find a CBLAS library") - endif () -endif () - -# Deprecated declarations. -set (NATIVE_CBLAS_INCLUDE_PATH ${CBLAS_INCLUDE_DIR} ) -get_filename_component (NATIVE_CBLAS_LIB_PATH ${CBLAS_LIBRARY} PATH) - -mark_as_advanced( - CBLAS_LIBRARY - CBLAS_INCLUDE_DIR - ) diff --git a/CMake/ARMA_FindCLAPACK.cmake b/CMake/ARMA_FindCLAPACK.cmake deleted file mode 100644 index 97a9792a31..0000000000 --- a/CMake/ARMA_FindCLAPACK.cmake +++ /dev/null @@ -1,48 +0,0 @@ -# - Find a version of CLAPACK (includes and library) -# This module defines -# CLAPACK_INCLUDE_DIR -# CLAPACK_LIBRARIES -# CLAPACK_FOUND -# also defined, but not for general use are -# CLAPACK_LIBRARY, where to find the library. - -find_path(CLAPACK_INCLUDE_DIR clapack.h -/usr/include/atlas/ -/usr/local/include/atlas/ -/usr/include/ -/usr/local/include/ -) - -set(CLAPACK_NAMES ${CLAPACK_NAMES} lapack_atlas) -set(CLAPACK_NAMES ${CLAPACK_NAMES} clapack) -find_library(CLAPACK_LIBRARY - NAMES ${CLAPACK_NAMES} - PATHS /usr/lib64/atlas-sse3 /usr/lib64/atlas /usr/lib64 /usr/local/lib64/atlas /usr/local/lib64 /usr/lib/atlas-sse3 /usr/lib/atlas-sse2 /usr/lib/atlas-sse /usr/lib/atlas-3dnow /usr/lib/atlas /usr/lib /usr/local/lib/atlas /usr/local/lib - ) - -if (CLAPACK_LIBRARY AND CLAPACK_INCLUDE_DIR) - set(CLAPACK_LIBRARIES ${CLAPACK_LIBRARY}) - set(CLAPACK_FOUND "YES") -else () - set(CLAPACK_FOUND "NO") -endif () - - -if (CLAPACK_FOUND) - if (NOT CLAPACK_FIND_QUIETLY) - message(STATUS "Found a CLAPACK library: ${CLAPACK_LIBRARIES}") - endif () -else () - if (CLAPACK_FIND_REQUIRED) - message(FATAL_ERROR "Could not find a CLAPACK library") - endif () -endif () - -# Deprecated declarations. -set (NATIVE_CLAPACK_INCLUDE_PATH ${CLAPACK_INCLUDE_DIR} ) -get_filename_component (NATIVE_CLAPACK_LIB_PATH ${CLAPACK_LIBRARY} PATH) - -mark_as_advanced( - CLAPACK_LIBRARY - CLAPACK_INCLUDE_DIR - ) diff --git a/CMake/ARMA_FindLAPACK.cmake b/CMake/ARMA_FindLAPACK.cmake deleted file mode 100644 index b40d16d04c..0000000000 --- a/CMake/ARMA_FindLAPACK.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# - Find a LAPACK library (no includes) -# This module defines -# LAPACK_LIBRARIES, the libraries needed to use LAPACK. -# LAPACK_FOUND, If false, do not try to use LAPACK. -# also defined, but not for general use are -# LAPACK_LIBRARY, where to find the LAPACK library. - -set(LAPACK_NAMES ${LAPACK_NAMES} lapack) - -# Check ATLAS paths preferentially, using this necessary hack (I love CMake). -find_library(LAPACK_LIBRARY - NAMES ${LAPACK_NAMES} - PATHS /usr/lib64/atlas /usr/lib/atlas /usr/local/lib64/atlas /usr/local/lib/atlas - NO_DEFAULT_PATH) - -find_library(LAPACK_LIBRARY - NAMES ${LAPACK_NAMES} - PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib - ) - -if (LAPACK_LIBRARY) - set(LAPACK_LIBRARIES ${LAPACK_LIBRARY}) - set(LAPACK_FOUND "YES") -else () - set(LAPACK_FOUND "NO") -endif () - - -if (LAPACK_FOUND) - if (NOT LAPACK_FIND_QUIETLY) - message(STATUS "Found LAPACK: ${LAPACK_LIBRARIES}") - endif () -else () - if (LAPACK_FIND_REQUIRED) - message(FATAL_ERROR "Could not find LAPACK") - endif () -endif () - -# Deprecated declarations. -get_filename_component (NATIVE_LAPACK_LIB_PATH ${LAPACK_LIBRARY} PATH) - -mark_as_advanced( - LAPACK_LIBRARY - ) diff --git a/CMake/ARMA_FindMKL.cmake b/CMake/ARMA_FindMKL.cmake deleted file mode 100644 index 452fa5a643..0000000000 --- a/CMake/ARMA_FindMKL.cmake +++ /dev/null @@ -1,49 +0,0 @@ -# - Find the MKL libraries (no includes) -# This module defines -# MKL_LIBRARIES, the libraries needed to use Intel's implementation of BLAS & LAPACK. -# MKL_FOUND, If false, do not try to use MKL. - -set(MKL_NAMES ${MKL_NAMES} mkl_lapack) -set(MKL_NAMES ${MKL_NAMES} mkl_intel_thread) -set(MKL_NAMES ${MKL_NAMES} mkl_core) -set(MKL_NAMES ${MKL_NAMES} guide) -set(MKL_NAMES ${MKL_NAMES} mkl) -set(MKL_NAMES ${MKL_NAMES} iomp5) -#set(MKL_NAMES ${MKL_NAMES} pthread) - -if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(MKL_NAMES ${MKL_NAMES} mkl_intel_lp64) -else() - set(MKL_NAMES ${MKL_NAMES} mkl_intel) -endif() - -foreach (MKL_NAME ${MKL_NAMES}) - find_library(${MKL_NAME}_LIBRARY - NAMES ${MKL_NAME} - PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib /opt/intel/lib/intel64 /opt/intel/lib/ia32 /opt/intel/mkl/lib/lib64 /opt/intel/mkl/lib/intel64 /opt/intel/mkl/lib/ia32 /opt/intel/mkl/lib /opt/intel/*/mkl/lib/intel64 /opt/intel/*/mkl/lib/ia32/ /opt/mkl/*/lib/em64t /opt/mkl/*/lib/32 /opt/intel/mkl/*/lib/em64t /opt/intel/mkl/*/lib/32 - ) - - set(TMP_LIBRARY ${${MKL_NAME}_LIBRARY}) - - if(TMP_LIBRARY) - set(MKL_LIBRARIES ${MKL_LIBRARIES} ${TMP_LIBRARY}) - endif() -endforeach() - -if (MKL_LIBRARIES) - set(MKL_FOUND "YES") -else () - set(MKL_FOUND "NO") -endif () - -if (MKL_FOUND) - if (NOT MKL_FIND_QUIETLY) - message(STATUS "Found MKL libraries: ${MKL_LIBRARIES}") - endif () -else () - if (MKL_FIND_REQUIRED) - message(FATAL_ERROR "Could not find MKL libraries") - endif () -endif () - -# mark_as_advanced(MKL_LIBRARY) diff --git a/CMake/ARMA_FindOpenBLAS.cmake b/CMake/ARMA_FindOpenBLAS.cmake deleted file mode 100644 index edfa27db83..0000000000 --- a/CMake/ARMA_FindOpenBLAS.cmake +++ /dev/null @@ -1,37 +0,0 @@ -# - Find the OpenBLAS library (no includes) -# This module defines -# OpenBLAS_LIBRARIES, the libraries needed to use OpenBLAS. -# OpenBLAS_FOUND, If false, do not try to use OpenBLAS. -# also defined, but not for general use are -# OpenBLAS_LIBRARY, where to find the OpenBLAS library. - -set(OpenBLAS_NAMES ${OpenBLAS_NAMES} openblas) -find_library(OpenBLAS_LIBRARY - NAMES ${OpenBLAS_NAMES} - PATHS /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib - ) - -if (OpenBLAS_LIBRARY) - set(OpenBLAS_LIBRARIES ${OpenBLAS_LIBRARY}) - set(OpenBLAS_FOUND "YES") -else () - set(OpenBLAS_FOUND "NO") -endif () - - -if (OpenBLAS_FOUND) - if (NOT OpenBLAS_FIND_QUIETLY) - message(STATUS "Found the OpenBLAS library: ${OpenBLAS_LIBRARIES}") - endif () -else () - if (OpenBLAS_FIND_REQUIRED) - message(FATAL_ERROR "Could not find the OpenBLAS library") - endif () -endif () - -# Deprecated declarations. -get_filename_component (NATIVE_OpenBLAS_LIB_PATH ${OpenBLAS_LIBRARY} PATH) - -mark_as_advanced( - OpenBLAS_LIBRARY - ) diff --git a/CMake/FindARPACK.cmake b/CMake/FindARPACK.cmake new file mode 100644 index 0000000000..08baa3b2b1 --- /dev/null +++ b/CMake/FindARPACK.cmake @@ -0,0 +1,55 @@ +# Searches for an installation of the ARPACK library. On success, it sets the following variables: +# +# ARPACK_FOUND Set to true to indicate the library was found +# ARPACK_LIBRARIES All libraries needed to use ARPACK (with full path) +# +# To specify an additional directory to search, set ARPACK_ROOT. +# +# TODO: Do we need to explicitly search for BLAS and LAPACK as well? The source distribution statically links these to +# libarpack. Are there any installations that don't do this or the equivalent? +# +# Author: Siddhartha Chaudhuri, 2009 +# + +SET(ARPACK_FOUND FALSE) + +# First look in user-provided root directory, then look in system locations +FIND_LIBRARY(ARPACK_LIBRARIES NAMES arpack libarpack ARPACK libARPACK PATHS "${ARPACK_ROOT}" "${ARPACK_ROOT}/lib" + NO_DEFAULT_PATH) +IF(NOT ARPACK_LIBRARIES) + FIND_LIBRARY(ARPACK_LIBRARIES NAMES arpack libarpack ARPACK libARPACK) +ENDIF(NOT ARPACK_LIBRARIES) + +IF(ARPACK_LIBRARIES) + # On OS X we probably also need gfortran and BLAS and LAPACK libraries + IF(APPLE) + FIND_LIBRARY(ARPACK_LAPACK_LIBRARY NAMES lapack LAPACK PATHS "${ARPACK_ROOT}" "${ARPACK_ROOT}/lib") + FIND_LIBRARY(ARPACK_BLAS_LIBRARY NAMES blas BLAS PATHS "${ARPACK_ROOT}" "${ARPACK_ROOT}/lib") + FIND_LIBRARY(ARPACK_GFORTRAN_LIBRARY NAMES gfortran PATHS "${ARPACK_ROOT}" "${ARPACK_ROOT}/lib" + PATH_SUFFIXES "" "gfortran/lib" "../gfortran/lib") + + IF(ARPACK_BLAS_LIBRARY) + SET(ARPACK_LIBRARIES ${ARPACK_LIBRARIES} ${ARPACK_BLAS_LIBRARY}) + ENDIF(ARPACK_BLAS_LIBRARY) + + IF(ARPACK_LAPACK_LIBRARY) + SET(ARPACK_LIBRARIES ${ARPACK_LIBRARIES} ${ARPACK_LAPACK_LIBRARY}) + ENDIF(ARPACK_LAPACK_LIBRARY) + + IF(ARPACK_GFORTRAN_LIBRARY) + SET(ARPACK_LIBRARIES ${ARPACK_LIBRARIES} ${ARPACK_GFORTRAN_LIBRARY}) + ENDIF(ARPACK_GFORTRAN_LIBRARY) + ENDIF(APPLE) + + SET(ARPACK_FOUND TRUE) +ENDIF(ARPACK_LIBRARIES) + +IF(ARPACK_FOUND) + IF(NOT ARPACK_FIND_QUIETLY) + MESSAGE(STATUS "Found ARPACK: libraries at ${ARPACK_LIBRARIES}") + ENDIF(NOT ARPACK_FIND_QUIETLY) +ELSE(ARPACK_FOUND) + IF(ARPACK_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "ARPACK not found") + ENDIF(ARPACK_FIND_REQUIRED) +ENDIF(ARPACK_FOUND) diff --git a/CMake/FindArmadillo.cmake b/CMake/FindArmadillo.cmake index 1244f64936..36ec63b515 100644 --- a/CMake/FindArmadillo.cmake +++ b/CMake/FindArmadillo.cmake @@ -1,41 +1,41 @@ -# - Find Armadillo -# Find the Armadillo C++ library -# -# Using Armadillo: -# find_package(Armadillo REQUIRED) -# include_directories(${ARMADILLO_INCLUDE_DIRS}) -# add_executable(foo foo.cc) -# target_link_libraries(foo ${ARMADILLO_LIBRARIES}) -# This module sets the following variables: -# ARMADILLO_FOUND - set to true if the library is found -# ARMADILLO_INCLUDE_DIRS - list of required include directories -# ARMADILLO_LIBRARIES - list of libraries to be linked -# ARMADILLO_VERSION_MAJOR - major version number -# ARMADILLO_VERSION_MINOR - minor version number -# ARMADILLO_VERSION_PATCH - patch version number -# ARMADILLO_VERSION_STRING - version number as a string (ex: "1.0.4") -# ARMADILLO_VERSION_NAME - name of the version (ex: "Antipodean Antileech") +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. -#============================================================================= -# Copyright 2011 Clement Creusot -# -# Distributed under the OSI-approved BSD License (the "License"); -# see accompanying file Copyright.txt for details. -# -# This software is distributed WITHOUT ANY WARRANTY; without even the -# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the License for more information. -#============================================================================= -# (To distribute this file outside of CMake, substitute the full -# License text for the above reference.) +#[=======================================================================[.rst: +FindArmadillo +------------- +Find the Armadillo C++ library. +Armadillo is a library for linear algebra & scientific computing. + +Using Armadillo: + +:: + + find_package(Armadillo REQUIRED) + include_directories(${ARMADILLO_INCLUDE_DIRS}) + add_executable(foo foo.cc) + target_link_libraries(foo ${ARMADILLO_LIBRARIES}) + +This module sets the following variables: + +:: + + ARMADILLO_FOUND - set to true if the library is found + ARMADILLO_INCLUDE_DIRS - list of required include directories + ARMADILLO_LIBRARIES - list of libraries to be linked + ARMADILLO_VERSION_MAJOR - major version number + ARMADILLO_VERSION_MINOR - minor version number + ARMADILLO_VERSION_PATCH - patch version number + ARMADILLO_VERSION_STRING - version number as a string (ex: "1.0.4") + ARMADILLO_VERSION_NAME - name of the version (ex: "Antipodean Antileech") +#]=======================================================================] find_path(ARMADILLO_INCLUDE_DIR NAMES armadillo PATHS "$ENV{ProgramFiles}/Armadillo/include" ) - if(ARMADILLO_INCLUDE_DIR) # ------------------------------------------------------------------------ # Extract version information from @@ -52,318 +52,117 @@ if(ARMADILLO_INCLUDE_DIR) if(EXISTS "${ARMADILLO_INCLUDE_DIR}/armadillo_bits/arma_version.hpp") # Read and parse armdillo version header file for version number - file(READ "${ARMADILLO_INCLUDE_DIR}/armadillo_bits/arma_version.hpp" _armadillo_HEADER_CONTENTS) - string(REGEX REPLACE ".*#define ARMA_VERSION_MAJOR ([0-9]+).*" "\\1" ARMADILLO_VERSION_MAJOR "${_armadillo_HEADER_CONTENTS}") - string(REGEX REPLACE ".*#define ARMA_VERSION_MINOR ([0-9]+).*" "\\1" ARMADILLO_VERSION_MINOR "${_armadillo_HEADER_CONTENTS}") - string(REGEX REPLACE ".*#define ARMA_VERSION_PATCH ([0-9]+).*" "\\1" ARMADILLO_VERSION_PATCH "${_armadillo_HEADER_CONTENTS}") + file(STRINGS "${ARMADILLO_INCLUDE_DIR}/armadillo_bits/arma_version.hpp" _ARMA_HEADER_CONTENTS REGEX "#define ARMA_VERSION_[A-Z]+ ") + string(REGEX REPLACE ".*#define ARMA_VERSION_MAJOR ([0-9]+).*" "\\1" ARMADILLO_VERSION_MAJOR "${_ARMA_HEADER_CONTENTS}") + string(REGEX REPLACE ".*#define ARMA_VERSION_MINOR ([0-9]+).*" "\\1" ARMADILLO_VERSION_MINOR "${_ARMA_HEADER_CONTENTS}") + string(REGEX REPLACE ".*#define ARMA_VERSION_PATCH ([0-9]+).*" "\\1" ARMADILLO_VERSION_PATCH "${_ARMA_HEADER_CONTENTS}") # WARNING: The number of spaces before the version name is not one. - string(REGEX REPLACE ".*#define ARMA_VERSION_NAME\ +\"([0-9a-zA-Z\ _-]+)\".*" "\\1" ARMADILLO_VERSION_NAME "${_armadillo_HEADER_CONTENTS}") + string(REGEX REPLACE ".*#define ARMA_VERSION_NAME\ +\"([0-9a-zA-Z\ _-]+)\".*" "\\1" ARMADILLO_VERSION_NAME "${_ARMA_HEADER_CONTENTS}") endif() set(ARMADILLO_VERSION_STRING "${ARMADILLO_VERSION_MAJOR}.${ARMADILLO_VERSION_MINOR}.${ARMADILLO_VERSION_PATCH}") endif () - -#====================== - -# Determine what support libraries are being used, and whether or not we need to -# link against them. We need to look in config.hpp. -set(SUPPORT_INCLUDE_DIRS "") -set(SUPPORT_LIBRARIES "") -set(ARMA_NEED_LIBRARY true) # Assume true. if(EXISTS "${ARMADILLO_INCLUDE_DIR}/armadillo_bits/config.hpp") - file(READ "${ARMADILLO_INCLUDE_DIR}/armadillo_bits/config.hpp" _armadillo_CONFIG_CONTENTS) - # ARMA_USE_WRAPPER - string(REGEX MATCH "\r?\n[\t ]*#define[ \t]+ARMA_USE_WRAPPER[ \t]*\r?\n" ARMA_USE_WRAPPER "${_armadillo_CONFIG_CONTENTS}") - - # ARMA_USE_LAPACK - string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]ARMA_USE_LAPACK[)][\t ]*\r?\n[\t ]*#define[ \t]+ARMA_USE_LAPACK[ \t]*\r?\n" ARMA_USE_LAPACK "${_armadillo_CONFIG_CONTENTS}") - - # ARMA_USE_BLAS - string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]ARMA_USE_BLAS[)][\t ]*\r?\n[\t ]*#define[ \t]+ARMA_USE_BLAS[ \t]*\r?\n" ARMA_USE_BLAS "${_armadillo_CONFIG_CONTENTS}") - # ARMA_USE_ARPACK - # ARMA_USE_ARPACK - string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]ARMA_USE_ARPACK[)][\t ]*\r?\n[\t ]*#define[ \t]+ARMA_USE_ARPACK[ \t]*\r?\n" ARMA_USE_ARPACK "${_armadillo_CONFIG_CONTENTS}") - - # Look for #define ARMA_USE_HDF5. - string(REGEX MATCH "\r?\n[\t ]*#if[\t ]+!defined[(]ARMA_USE_HDF5[)][\t ]*\r?\n[\t ]*#define[ \t]+ARMA_USE_HDF5[ \t]*\r?\n" ARMA_USE_HDF5 "${_armadillo_CONFIG_CONTENTS}") - - # If we aren't wrapping, things get a little more complex. - if("${ARMA_USE_WRAPPER}" STREQUAL "") - set(ARMA_NEED_LIBRARY false) - message(STATUS "ARMA_USE_WRAPPER is not defined, so all dependencies of " - "Armadillo must be manually linked.") - - set(HAVE_LAPACK false) - set(HAVE_BLAS false) - - # Search for LAPACK/BLAS (or replacement). - if ((NOT "${ARMA_USE_LAPACK}" STREQUAL "") AND - (NOT "${ARMA_USE_BLAS}" STREQUAL "")) - # In order of preference: MKL, ACML, OpenBLAS, ATLAS - set(MKL_FIND_QUIETLY true) - include(ARMA_FindMKL) - set(ACMLMP_FIND_QUIETLY true) - include(ARMA_FindACMLMP) - set(ACML_FIND_QUIETLY true) - include(ARMA_FindACML) - - if (MKL_FOUND) - message(STATUS "Using MKL for LAPACK/BLAS: ${MKL_LIBRARIES}") - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${MKL_LIBRARIES}") - set(HAVE_LAPACK true) - set(HAVE_BLAS true) - elseif (ACMLMP_FOUND) - message(STATUS "Using multi-core ACML libraries for LAPACK/BLAS: - ${ACMLMP_LIBRARIES}") - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${ACMLMP_LIBRARIES}") - set(HAVE_LAPACK true) - set(HAVE_BLAS true) - elseif (ACML_FOUND) - message(STATUS "Using ACML for LAPACK/BLAS: ${ACML_LIBRARIES}") - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${ACML_LIBRARIES}") - set(HAVE_LAPACK true) - set(HAVE_BLAS true) - endif () - endif () - - # If we haven't found BLAS, try. - if (NOT "${ARMA_USE_BLAS}" STREQUAL "" AND NOT HAVE_BLAS) - # Search for BLAS. - set(OpenBLAS_FIND_QUIETLY true) - include(ARMA_FindOpenBLAS) - set(CBLAS_FIND_QUIETLY true) - include(ARMA_FindCBLAS) - set(BLAS_FIND_QUIETLY true) - include(ARMA_FindBLAS) - - if (OpenBLAS_FOUND) - # Warn if ATLAS is found also. - if (CBLAS_FOUND) - message(STATUS "Warning: both OpenBLAS and ATLAS have been found; " - "ATLAS will not be used.") - endif () - message(STATUS "Using OpenBLAS for BLAS: ${OpenBLAS_LIBRARIES}") - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${OpenBLAS_LIBRARIES}") - set(HAVE_BLAS true) - elseif (CBLAS_FOUND) - message(STATUS "Using ATLAS for BLAS: ${CBLAS_LIBRARIES}") - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${CBLAS_LIBRARIES}") - set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}" - "${CBLAS_INCLUDE_DIR}") - set(HAVE_BLAS true) - elseif (BLAS_FOUND) - message(STATUS "Using standard BLAS: ${BLAS_LIBRARIES}") - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${BLAS_LIBRARIES}") - set(HAVE_BLAS true) - endif () - endif () - - # If we haven't found LAPACK, try. - if (NOT "${ARMA_USE_LAPACK}" STREQUAL "" AND NOT HAVE_LAPACK) - # Search for LAPACK. - set(CLAPACK_FIND_QUIETLY true) - include(ARMA_FindCLAPACK) - set(LAPACK_FIND_QUIETLY true) - include(ARMA_FindLAPACK) - - # Only use ATLAS if OpenBLAS isn't being used. - if (CLAPACK_FOUND AND NOT OpenBLAS_FOUND) - message(STATUS "Using ATLAS for LAPACK: ${CLAPACK_LIBRARIES}") - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${CLAPACK_LIBRARIES}") - set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}" - "${CLAPACK_INCLUDE_DIR}") - set(HAVE_LAPACK true) - elseif (LAPACK_FOUND) - message(STATUS "Using standard LAPACK: ${LAPACK_LIBRARIES}") - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${LAPACK_LIBRARIES}") - set(HAVE_LAPACK true) - endif () - endif () - - if (NOT "${ARMA_USE_LAPACK}" STREQUAL "" AND NOT HAVE_LAPACK) - message(FATAL_ERROR "Cannot find LAPACK library, but ARMA_USE_LAPACK is " - "set. Try specifying LAPACK libraries manually by setting the " - "LAPACK_LIBRARY variable.") - endif () - - if (NOT "${ARMA_USE_BLAS}" STREQUAL "" AND NOT HAVE_BLAS) - message(FATAL_ERROR "Cannot find BLAS library, but ARMA_USE_BLAS is set. " - "Try specifying BLAS libraries manually by setting the BLAS_LIBRARY " - "variable.") - endif () - - # Search for ARPACK (or replacement). - if (NOT "${ARMA_USE_ARPACK}" STREQUAL "") - # Use Armadillo ARPACK-finding procedure. - set(ARPACK_FIND_QUIETLY true) - include(ARMA_FindARPACK) - - if (NOT ARPACK_FOUND) - message(FATAL_ERROR "ARMA_USE_ARPACK is defined in " - "armadillo_bits/config.hpp, but ARPACK cannot be found. Try " - "specifying ARPACK_LIBRARY.") - endif () - - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${ARPACK_LIBRARY}") - endif () - - # Search for HDF5 (or replacement). - if (NOT "${ARMA_USE_HDF5}" STREQUAL "") - find_package(HDF5 QUIET) - - if(NOT HDF5_FOUND) - # On Debian systems, the HDF5 package has been split into multiple - # packages so that it is co-installable. But this may mean that the - # include files are hidden somewhere very odd that the FindHDF5.cmake - # script will not find. Thus, we'll also quickly check pkgconfig to see - # if there is information on what to use there. - find_package(PkgConfig) - if (PKG_CONFIG_FOUND) - pkg_check_modules(HDF5 hdf5) - # But using pkgconfig is a little weird because HDF5_LIBRARIES won't - # be filled with exact library paths, like the other scripts. So - # instead what we get is HDF5_LIBRARY_DIRS which is the equivalent of - # what we'd pass to -L. - if (HDF5_FOUND) - # I'm not sure what I think of doing this here... - link_directories("${HDF5_LIBRARY_DIRS}") - endif() - endif() - endif() - - if(NOT HDF5_FOUND) - # We tried but didn't find it. - message(FATAL_ERROR "Armadillo HDF5 support is enabled, but HDF5 " - "cannot be found on the system. Consider disabling HDF5 support.") - endif() - - set(SUPPORT_INCLUDE_DIRS "${SUPPORT_INCLUDE_DIRS}" "${HDF5_INCLUDE_DIRS}") - set(SUPPORT_LIBRARIES "${SUPPORT_LIBRARIES}" "${HDF5_LIBRARIES}") - endif () - - else() - # Some older versions still require linking against HDF5 since they did not - # wrap libhdf5. This was true for versions older than 4.300. - if(NOT "${ARMA_USE_HDF5}" STREQUAL "" AND - "${ARMADILLO_VERSION_STRING}" VERSION_LESS "4.300.0") - message(STATUS "Armadillo HDF5 support is enabled and manual linking is " - "required.") - # We have HDF5 support and need to link against HDF5. - find_package(HDF5) - - if(NOT HDF5_FOUND) - # On Debian systems, the HDF5 package has been split into multiple - # packages so that it is co-installable. But this may mean that the - # include files are hidden somewhere very odd that the FindHDF5.cmake - # script will not find. Thus, we'll also quickly check pkgconfig to see - # if there is information on what to use there. - find_package(PkgConfig) - if (PKG_CONFIG_FOUND) - pkg_check_modules(HDF5 hdf5) - # But using pkgconfig is a little weird because HDF5_LIBRARIES won't - # be filled with exact library paths, like the other scripts. So - # instead what we get is HDF5_LIBRARY_DIRS which is the equivalent of - # what we'd pass to -L. - if (HDF5_FOUND) - # I'm not sure what I think of doing this here... - link_directories("${HDF5_LIBRARY_DIRS}") - endif() - endif() - endif() - - if(NOT HDF5_FOUND) - # We tried but didn't find it. - message(FATAL_ERROR "Armadillo HDF5 support is enabled, but HDF5 " - "cannot be found on the system. Consider disabling HDF5 support.") - endif() - - set(SUPPORT_INCLUDE_DIRS "${HDF5_INCLUDE_DIRS}") - set(SUPPORT_LIBRARIES "${HDF5_LIBRARIES}") - endif() - - # Versions between 4.300 and 4.500 did successfully wrap HDF5, but didn't have good support for setting the include directory correctly. - if(NOT "${ARMA_USE_HDF5}" STREQUAL "" AND - "${ARMADILLO_VERSION_STRING}" VERSION_GREATER "4.299.0" AND - "${ARMADILLO_VERSION_STRING}" VERSION_LESS "4.450.0") - message(STATUS "Armadillo HDF5 support is enabled and include " - "directories must be found.") - find_package(HDF5) - - if(NOT HDF5_FOUND) - # On Debian systems, the HDF5 package has been split into multiple - # packages so that it is co-installable. But this may mean that the - # include files are hidden somewhere very odd that the FindHDF5.cmake - # script will not find. Thus, we'll also quickly check pkgconfig to see - # if there is information on what to use there. - find_package(PkgConfig) - if (PKG_CONFIG_FOUND) - pkg_check_modules(HDF5 hdf5) - endif() - endif() - - if(NOT HDF5_FOUND) - # We tried but didn't find it. - message(FATAL_ERROR "Armadillo HDF5 support is enabled, but HDF5 " - "cannot be found on the system. Consider disabling HDF5 support.") - endif() - - set(SUPPORT_INCLUDE_DIRS "${HDF5_INCLUDE_DIRS}") - endif() - - endif() -else() - message(FATAL_ERROR "${ARMADILLO_INCLUDE_DIR}/armadillo_bits/config.hpp not " - "found! Cannot determine what to link against.") + file(STRINGS "${ARMADILLO_INCLUDE_DIR}/armadillo_bits/config.hpp" _ARMA_CONFIG_CONTENTS REGEX "^#define ARMA_USE_[A-Z]+") + string(REGEX MATCH "ARMA_USE_WRAPPER" _ARMA_USE_WRAPPER "${_ARMA_CONFIG_CONTENTS}") + string(REGEX MATCH "ARMA_USE_LAPACK" _ARMA_USE_LAPACK "${_ARMA_CONFIG_CONTENTS}") + string(REGEX MATCH "ARMA_USE_BLAS" _ARMA_USE_BLAS "${_ARMA_CONFIG_CONTENTS}") + string(REGEX MATCH "ARMA_USE_ARPACK" _ARMA_USE_ARPACK "${_ARMA_CONFIG_CONTENTS}") + string(REGEX MATCH "ARMA_USE_HDF5" _ARMA_USE_HDF5 "${_ARMA_CONFIG_CONTENTS}") endif() -if (ARMA_NEED_LIBRARY) +include(FindPackageHandleStandardArgs) + +# If _ARMA_USE_WRAPPER is set, then we just link to armadillo, but if it's not then we need support libraries instead +set(_ARMA_SUPPORT_LIBRARIES) + +if(_ARMA_USE_WRAPPER) # UNIX paths are standard, no need to write. find_library(ARMADILLO_LIBRARY NAMES armadillo PATHS "$ENV{ProgramFiles}/Armadillo/lib" "$ENV{ProgramFiles}/Armadillo/lib64" "$ENV{ProgramFiles}/Armadillo" ) + set(_ARMA_REQUIRED_VARS ARMADILLO_LIBRARY ARMADILLO_INCLUDE_DIR VERSION_VAR ARMADILLO_VERSION_STRING) +else() + # don't link to armadillo in this case + set(ARMADILLO_LIBRARY "") + if(_ARMA_USE_LAPACK) + if(ARMADILLO_FIND_QUIETLY OR NOT ARMADILLO_FIND_REQUIRED) + find_package(LAPACK QUIET) + else() + find_package(LAPCK REQUIRED) + endif() + if(LAPACK_FOUND) + set(_ARMA_SUPPORT_LIBRARIES "${_ARMA_SUPPORT_LIBRARIES}" "${LAPACK_LIBRARIES}") + endif() + endif() + if(_ARMA_USE_BLAS) + if(ARMADILLO_FIND_QUIETLY OR NOT ARMADILLO_FIND_REQUIRED) + find_package(BLAS QUIET) + else() + find_package(BLAS REQUIRED) + endif() + if(BLAS_FOUND) + set(_ARMA_SUPPORT_LIBRARIES "${_ARMA_SUPPORT_LIBRARIES}" "${BLAS_LIBRARIES}") + endif() + endif() + if(_ARMA_USE_ARPACK) + if(ARMADILLO_FIND_QUIETLY OR NOT ARMADILLO_FIND_REQUIRED) + find_package(ARPACK QUIET) + else() + find_package(ARPACK REQUIRED) + endif() + if(ARPACK_FOUND) + set(_ARMA_SUPPORT_LIBRARIES "${_ARMA_SUPPORT_LIBRARIES}" "${ARPACK_LIBRARIES}") + endif() + endif() + if(_ARMA_USE_HDF5) + find_package(HDF5 QUIET) + if(NOT HDF5_FOUND) + # On Debian systems, the HDF5 package has been split into multiple + # packages so that it is co-installable. But this may mean that the + # include files are hidden somewhere very odd that FindHDF5.cmake will + # not find. Thus, we'll also quickly check pkgconfig to see if there is + # information on what to use there. + message(WARNING "HDF5 required but not found; using PkgConfig") + find_package(PkgConfig) + if (PKG_CONFIG_FOUND) + pkg_check_modules(HDF5 REQUIRED hdf5) + link_directories("${HDF5_LIBRARY_DIRS}") + else() + message(FATAL_ERROR "PkgConfig (Used to help find HDF5) was not found") + endif() + endif() + set(_ARMA_SUPPORT_INCLUDE_DIRS "${HDF5_INCLUDE_DIRS}") + set(_ARMA_SUPPORT_LIBRARIES "${_ARMA_SUPPORT_LIBRARIES}" "${HDF5_LIBRARIES}") + endif() + set(ARMADILLO_FOUND true) + set(_ARMA_REQUIRED_VARS ARMADILLO_INCLUDE_DIR VERSION_VAR ARMADILLO_VERSION_STRING) +endif() - # Checks 'REQUIRED', 'QUIET' and versions. - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(Armadillo - REQUIRED_VARS ARMADILLO_LIBRARY ARMADILLO_INCLUDE_DIR - VERSION_VAR ARMADILLO_VERSION_STRING) - # version_var fails with cmake < 2.8.4. -else () - # Checks 'REQUIRED', 'QUIET' and versions. - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(Armadillo - REQUIRED_VARS ARMADILLO_INCLUDE_DIR - VERSION_VAR ARMADILLO_VERSION_STRING) -endif () +find_package_handle_standard_args(Armadillo REQUIRED_VARS ${_ARMA_REQUIRED_VARS}) if (ARMADILLO_FOUND) - # Also include support include directories. - set(ARMADILLO_INCLUDE_DIRS ${ARMADILLO_INCLUDE_DIR} ${SUPPORT_INCLUDE_DIRS}) - # Also include support libraries to link against. - if (ARMA_NEED_LIBRARY) - set(ARMADILLO_LIBRARIES ${ARMADILLO_LIBRARY} ${SUPPORT_LIBRARIES}) - else () - set(ARMADILLO_LIBRARIES ${SUPPORT_LIBRARIES}) - endif () - message(STATUS "Armadillo libraries: ${ARMADILLO_LIBRARIES}") + set(ARMADILLO_INCLUDE_DIRS ${ARMADILLO_INCLUDE_DIR}) + set(ARMADILLO_LIBRARIES ${ARMADILLO_LIBRARY} ${_ARMA_SUPPORT_LIBRARIES}) endif () +# Clean up internal variables +unset(_ARMA_REQUIRED_VARS) +unset(_ARMA_SUPPORT_LIBRARIES) +unset(_ARMA_USE_WRAPPER) +unset(_ARMA_USE_LAPACK) +unset(_ARMA_USE_BLAS) +unset(_ARMA_USE_ARPACK) +unset(_ARMA_USE_HDF5) +unset(_ARMA_CONFIG_CONTENTS) +unset(_ARMA_HEADER_CONTENTS) +unset(__ARMA_SUPPORT_INCLUDE_DIRS) # Hide internal variables mark_as_advanced( ARMADILLO_INCLUDE_DIR ARMADILLO_LIBRARY) - -#====================== diff --git a/CMakeLists.txt b/CMakeLists.txt index 84e96b4510..4dc6d0beb8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,51 +292,6 @@ else() endif () endif() -# On Windows, Armadillo should be using LAPACK and BLAS but we still need to -# link against it. We don't want to use the FindLAPACK or FindBLAS modules -# because then we are required to have a FORTRAN compiler (argh!) so we will try -# and find LAPACK and BLAS ourselves, using a slightly modified variant of the -# script Armadillo uses to find these. -if (WIN32) - find_library(LAPACK_LIBRARY - NAMES lapack liblapack lapack_win32_MT lapack_win32 - PATHS "C:/Program Files/Armadillo" - PATH_SUFFIXES "examples/lib_win32/") - - if (NOT LAPACK_LIBRARY) - message(FATAL_ERROR "Cannot find LAPACK library (.lib)!") - endif () - - find_library(BLAS_LIBRARY - NAMES blas libblas blas_win32_MT blas_win32 - PATHS "C:/Program Files/Armadillo" - PATH_SUFFIXES "examples/lib_win32/") - - if (NOT BLAS_LIBRARY) - message(FATAL_ERROR "Cannot find BLAS library (.lib)!") - endif () - - # Piggyback LAPACK and BLAS linking into Armadillo link. - set(ARMADILLO_LIBRARIES - ${ARMADILLO_LIBRARIES} ${BLAS_LIBRARY} ${LAPACK_LIBRARY}) - - # Ensure that the libraries are added to the MSVC IDE runtime path. - get_filename_component(BLAS_DIR ${BLAS_LIBRARY} DIRECTORY) - get_filename_component(LAPACK_DIR ${LAPACK_LIBRARY} DIRECTORY) - - # Sometimes, especially with an OpenBLAS install via nuget, the DLLs are - # actually in ../../bin/x64/. Automatically add these. - if (EXISTS "${BLAS_DIR}/../../bin/x64/") - get_filename_component(BLAS_DLL_DIR "${BLAS_DIR}/../../bin/x64" ABSOLUTE) - set(DLL_COPY_DIRS ${DLL_COPY_DIRS} "${BLAS_DLL_DIR}") - endif () - - if (EXISTS "${LAPACK_DIR}/../../bin/x64/") - get_filename_component(LAPACK_DLL_DIR "${LAPACK_DIR}/../../bin/x64" ABSOLUTE) - set(DLL_COPY_DIRS ${DLL_COPY_DIRS} "${BLAS_DLL_DIR}") - endif () -endif () - # Include directories for the previous dependencies. set(MLPACK_INCLUDE_DIRS ${MLPACK_INCLUDE_DIRS} ${ARMADILLO_INCLUDE_DIRS}) set(MLPACK_LIBRARIES ${MLPACK_LIBRARIES} ${ARMADILLO_LIBRARIES}) @@ -373,6 +328,7 @@ if (NOT STB_IMAGE_FOUND) install(FILES "${CMAKE_BINARY_DIR}/deps/${STB_DIR}/stb_image.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") install(FILES "${CMAKE_BINARY_DIR}/deps/${STB_DIR}/stb_image_write.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") add_definitions(-DHAS_STB) + set(STB_AVAILABLE "1") else () message(WARNING "stb/stb_image.h is not installed. Image utilities will not be available!") @@ -393,6 +349,7 @@ else () # Already has STB installed. add_definitions(-DHAS_STB) set(MLPACK_INCLUDE_DIRS ${MLPACK_INCLUDE_DIRS} ${STB_IMAGE_INCLUDE_DIR}) + set(STB_AVAILABLE "1") endif () @@ -415,7 +372,12 @@ if (NOT ENSMALLEN_FOUND) # Get the name of the directory. file (GLOB ENS_DIRECTORIES RELATIVE "${CMAKE_BINARY_DIR}/deps/" "${CMAKE_BINARY_DIR}/deps/ensmallen-[0-9]*.[0-9]*.[0-9]*") - list(FILTER ENS_DIRECTORIES EXCLUDE REGEX "ensmallen-.*\.tar\.gz") + # list(FILTER) is not available on 3.5 or older, but try to keep + # configuring without filtering the list anyway (it might work if only + # the file ensmallen-latest.tar.gz is present. + if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.6.0") + list(FILTER ENS_DIRECTORIES EXCLUDE REGEX "ensmallen-.*\.tar\.gz") + endif () list(LENGTH ENS_DIRECTORIES ENS_DIRECTORIES_LEN) if (ENS_DIRECTORIES_LEN EQUAL 1) list(GET ENS_DIRECTORIES 0 ENSMALLEN_INCLUDE_DIR) @@ -453,6 +415,9 @@ endif () # Unfortunately this configuration variable is necessary and will need to be # updated as time goes on and new versions are released. set(Boost_ADDITIONAL_VERSIONS + "1.72.0" "1.72" + "1.71.0" "1.71" + "1.70.0" "1.70" "1.69.0" "1.69" "1.68.0" "1.68" "1.67.0" "1.67" @@ -464,23 +429,14 @@ set(Boost_ADDITIONAL_VERSIONS "1.61.1" "1.61.0" "1.61" "1.60.1" "1.60.0" "1.60" "1.59.1" "1.59.0" "1.59" - "1.58.1" "1.58.0" "1.58" - "1.57.1" "1.57.0" "1.57" - "1.56.1" "1.56.0" "1.56" - "1.55.1" "1.55.0" "1.55" - "1.54.1" "1.54.0" "1.54" - "1.53.1" "1.53.0" "1.53" - "1.52.1" "1.52.0" "1.52" - "1.51.1" "1.51.0" "1.51" - "1.50.1" "1.50.0" "1.50" - "1.49.1" "1.49.0" "1.49") + "1.58.1" "1.58.0" "1.58") # Disable forced config-mode CMake search for Boost, which only imports targets # and does not set the variables that we need. # # TODO for the brave: transition all mlpack's CMake to 'target-based modern # CMake'. Good luck! You'll need it. set(Boost_NO_BOOST_CMAKE 1) -find_package(Boost 1.49 +find_package(Boost 1.58 COMPONENTS program_options unit_test_framework diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b377a24e69..c9ac5e5aa7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,3 +28,49 @@ Members of the Contributors team are encouraged to review pull requests that have already been reviewed, and pull request contributors are encouraged to seek multiple reviews. Reviews from anyone not on the Contributors team are always appreciated and encouraged! + +## Reviewing Pull Requests + +All mlpack contributors who choose to review and provide feedback on pull +requests have a responsibility to both the project and the individual making +the contribution. + +Reviews and feedback should be +[helpful, insightful, and geared towards improving the contribution]( + https://www.youtube.com/watch?v=NNXk_WJzyMI). +If there are reasons why you feel the PR should not be merged, explain +what those are. Be open to having your mind changed. Be open to +working with the contributor to make the pull request better. + +Please don't leave dismissive or disrespectful reviews! It's not helpful for +anyone. + +When reviewing a pull request, the primary goals are: + +- For the codebase/project to improve +- For the person submitting the request to succeed + +Even if a pull request does not get merged, the submitters should come away +from the experience feeling like their effort was not wasted or unappreciated. +Every pull request from a new contributor is an opportunity to grow the community. + +When changes are necessary, request them, do not demand them, and do not assume +that the contributor already knows how to do that. Be there to lend a helping +hand in case of need. + +Since there can sometimes be a lot more pull requests being opened than +reviewed, we highly encourage everyone to review each others pull request +keeping in mind all the above mentioned points. + +Let's welcome new contributors with ❤️. + +## Pull Request Waiting Time + +mlpack is a community-driven project, so everyone only works on it in their +free time; this means it may take some time for them to review pull requests. +While gentle reminders are welcome, please be patient and avoid constantly +messaging contributors or tagging them on pull requests. + +Typically small PRs will be reviewed within a handful of days; larger PRs might +take a few weeks for an initial review, and it may be a little bit longer in +times of high activity. diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 60fba03e0a..db423bb3b8 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -128,6 +128,9 @@ Copyright: Copyright 2020, Sriram S K Copyright 2020, Manoranjan Kumar Bharti ( Nakul Bharti ) Copyright 2020, Saraansh Tandon + Copyright 2020, Gaurav Singh + Copyright 2020, Lakshya Ojha + Copyright 2020, Bisakh Mondal License: BSD-3-clause All rights reserved. diff --git a/HISTORY.md b/HISTORY.md index 6e4798c57b..6d086bf76f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,10 +1,14 @@ ### mlpack ?.?.? ###### ????-??-?? + * Templated return type of `Forward function` of loss functions (#2339). + + * Added `R2 Score` regression metric (#2323). + * Added `mean squared logarithmic error` loss function for neural networks (#2210). - + * Added `mean bias loss function` for neural networks (#2210). - + * The DecisionStump class has been marked deprecated; use the `DecisionTree` class with `NoRecursion=true` or use `ID3DecisionStump` instead (#2099). @@ -25,12 +29,18 @@ * Add functions to access parameters of `Convolution` and `AtrousConvolution` layers (#1985). + * Add Compute Error function in lars regression and changing Train function to + return computed error (#2139). + * Add Julia bindings (#1949). Build settings can be controlled with the `BUILD_JULIA_BINDINGS=(ON/OFF)` and `JULIA_EXECUTABLE=/path/to/julia` CMake parameters. * CMake fix for finding STB include directory (#2145). + * Add bindings for loading and saving images (#2019); `mlpack_image_converter` + from the command-line, `mlpack.image_converter()` from Python. + * Add normalization support for CF binding (#2136). * Add Mish activation function (#2158). @@ -42,12 +52,32 @@ * Better error handling of eigendecompositions and Cholesky decompositions (#2088, #1840). - + * Add LiSHT activation function (#2182). * Add Valid and Same Padding for Transposed Convolution layer (#2163). + + * Add CELU activation function (#2191) - * Add Log-Hyperbolic-Cosine Loss function (#2207). + * Add Log-Hyperbolic-Cosine Loss function (#2207) + + * Change neural network types to avoid unnecessary use of rvalue references + (#2259). + + * Bump minimum Boost version to 1.58 (#2305). + + * Refactor STB support so HAS_STB macro is not needed when compiling against + mlpack (#2312). + + * Add Hard Shrink Activation Function (#2186). + + * Add Soft Shrink Activation Function (#2174). + + * Add Hinge Embedding Loss Function (#2229). + + * Add Cosine Embedding Loss Function (#2209). + + * Add Margin Ranking Loss Function (#2264). ### mlpack 3.2.2 ###### 2019-11-26 diff --git a/README.md b/README.md index da9d26a9c8..d204bb4b3a 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ src="https://cdn.rawgit.com/mlpack/mlpack.org/e7d36ed8/mlpack-black.svg" style=" Jenkins Coveralls License + NumFOCUS

@@ -33,6 +34,8 @@ functions as a "swiss army knife" for machine learning researchers. In addition to its powerful C++ interface, mlpack also provides command-line programs, Python bindings, and Julia bindings. +[//]: # (numfocus-fiscal-sponsor-attribution) + mlpack uses an [open governance model](./GOVERNANCE.md) and is fiscally sponsored by [NumFOCUS](https://numfocus.org/). Consider making a [tax-deductible donation](https://numfocus.org/donate-to-mlpack) to help the @@ -40,7 +43,7 @@ project pay for developer time, professional services, travel, workshops, and a variety of other needs.

- + @@ -97,11 +100,11 @@ Citations are beneficial for the growth and improvement of mlpack. mlpack has the following dependencies: - Armadillo >= 8.400.0 + Armadillo >= 8.400.0 Boost (program_options, math_c99, unit_test_framework, serialization, - spirit) - CMake >= 3.3.2 - ensmallen >= 2.10.0 + spirit) >= 1.58.0 + CMake >= 3.3.2 + ensmallen >= 2.10.0 All of those should be available in your distribution's package manager. If not, you will have to compile each of them by hand. See the documentation for diff --git a/doc/guide/build.hpp b/doc/guide/build.hpp index e0577c39ba..df5e2edb82 100644 --- a/doc/guide/build.hpp +++ b/doc/guide/build.hpp @@ -79,7 +79,7 @@ system and have headers present: - Armadillo >= 8.400.0 (with LAPACK support) - Boost (math_c99, program_options, serialization, unit_test_framework, heap, - spirit) >= 1.49 + spirit) >= 1.58 - ensmallen >= 2.10.0 (will be downloaded if not found) In addition, mlpack has the following optional dependencies: diff --git a/doc/guide/build_windows.hpp b/doc/guide/build_windows.hpp index 469b866066..bb8ce9a1ea 100644 --- a/doc/guide/build_windows.hpp +++ b/doc/guide/build_windows.hpp @@ -118,7 +118,7 @@ compiler version, check if the Visual Studio compiler and Windows SDK are instal - Run cmake: @code -cmake -G "Visual Studio 16 2019" -A x64 -DBLAS_LIBRARY:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARY:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DARMADILLO_INCLUDE_DIR="C:/mlpack/armadillo/include" -DARMADILLO_LIBRARY:FILEPATH="C:/mlpack/armadillo/build/Debug/armadillo.lib" -DBOOST_INCLUDEDIR:PATH="C:/boost/" -DBOOST_LIBRARYDIR:PATH="C:/boost/lib64-msvc-14.2" -DDEBUG=OFF -DPROFILE=OFF .. +cmake -G "Visual Studio 16 2019" -A x64 -DBLAS_LIBRARIES:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARIES:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DARMADILLO_INCLUDE_DIR="C:/mlpack/armadillo/include" -DARMADILLO_LIBRARY:FILEPATH="C:/mlpack/armadillo/build/Debug/armadillo.lib" -DBOOST_INCLUDEDIR:PATH="C:/boost/" -DBOOST_LIBRARYDIR:PATH="C:/boost/lib64-msvc-14.2" -DDEBUG=OFF -DPROFILE=OFF .. @endcode @note cmake will attempt to automatically download the ensmallen dependency. If for some reason cmake can't download the dependency, you will need to manually download ensmallen from http://ensmallen.org/ and extract it to "C:\mlpack\mlpack\deps\". Then, specify the path to ensmallen using the flag: -DENSMALLEN_INCLUDE_DIR=C:/mlpack/mlpack/deps/ensmallen/include diff --git a/doc/guide/formats.hpp b/doc/guide/formats.hpp index acf8a00c20..e7f216aae3 100644 --- a/doc/guide/formats.hpp +++ b/doc/guide/formats.hpp @@ -330,39 +330,46 @@ mlpack's image saving/loading functionality is based on [stb/](https://github.co Image utilities supports loading and saving of images. -It supports filetypes "jpg", "png", "tga","bmp", "psd", "gif", "hdr", "pic", "pnm" for loading and "jpg", "png", "tga", "bmp", "hdr" for saving. +It supports filetypes "jpg", "png", "tga", "bmp", "psd", "gif", "hdr", "pic", +"pnm" for loading and "jpg", "png", "tga", "bmp", "hdr" for saving. -The datatype associated is unsigned char to support RGB values in the range 1-255. To feed data into the network typecast of `arma::Mat` may be required. Images are stored in matrix as (width * height * channels, NumberOfImages). Therefore imageMatrix.col(0) would be the first image if images are loaded in imageMatrix. +The datatype associated is unsigned char to support RGB values in the range +1-255. To feed data into the network typecast of `arma::Mat` may be required. +Images are stored in the matrix as (width * height * channels, NumberOfImages). +Therefore @c imageMatrix.col(0) would be the first image if images are loaded in +@c imageMatrix. @section imageinfo_api_imagetut Accessing Metadata of Images: ImageInfo ImageInfo class contains the metadata of the images. @code ImageInfo(const size_t width, - const size_t height, - const size_t channels); + const size_t height, + const size_t channels, + const size_t quality = 90); @endcode -Other public memebers include: - - flipVertical Flip the image vertical upon loading. - - quality Compression of the image if saved as jpg (0-100). + +The @c quality member denotes the compression of the image if it is saved as +`jpg`; it takes values from 0 to 100. @section load_api_imagetut Loading Images in C++ - Standalone loading of images. + @code - template - bool Load(const std::string& filename, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal, - const bool transpose); +template +bool Load(const std::string& filename, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal); @endcode -Loading a test image. It also fills up the ImageInfo class object. +The example below loads a test image. It also fills up the ImageInfo class +object. + @code data::ImageInfo info; -data::Load("test_image.png", matrix, info, false, true); +data::Load("test_image.png", matrix, info, false); @endcode ImageInfo requires height, width, number of channels of the image. @@ -377,18 +384,17 @@ More than one image can be loaded into the same matrix. Loading multiple images: @code - template - bool Load(const std::vector& files, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal, - const bool transpose); +template +bool Load(const std::vector& files, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal); @endcode @code - data::ImageInfo info; - std::vector> files{"test_image1.bmp","test_image2.bmp"}; - data::load(files, matrix, info, false, true); +data::ImageInfo info; +std::vector> files{"test_image1.bmp","test_image2.bmp"}; +data::Load(files, matrix, info, false); @endcode @section save_api_imagetut Saving Images in C++ diff --git a/doc/tutorials/ann/ann.txt b/doc/tutorials/ann/ann.txt index c1e7da31c1..db28bef868 100644 --- a/doc/tutorials/ann/ann.txt +++ b/doc/tutorials/ann/ann.txt @@ -204,17 +204,17 @@ using namespace mlpack::ann; int main() { - // Load the training set. - arma::mat dataset; - data::Load("thyroid_train.csv", dataset, true); + // Load the training set and testing set. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); - // Split the labels from the training set. - arma::mat trainData = dataset.submat(0, 0, dataset.n_rows - 4, - dataset.n_cols - 1); - - // Split the data from the training set. - arma::mat trainLabels = dataset.submat(dataset.n_rows - 3, 0, - dataset.n_rows - 1, dataset.n_cols - 1); + // Split the labels from the training set and testing set respectively. + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + arma::mat testLabels = testData.row(testData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); // Initialize the network. FFN<> model; @@ -226,14 +226,48 @@ int main() // Train the model. model.Train(trainData, trainLabels); - // Use the Predict method to get the assignments. - arma::mat assignments; - model.Predict(trainData, assignments); + // Use the Predict method to get the predictions. + arma::mat predictionTemp; + model.Predict(testData, predictionTemp); + + /* + Since the predictionsTemp is of dimensions (3 x number_of_data_points) + with continuous values, we first need to reduce it to a dimension of + (1 x number_of_data_points) with scalar values, to be able to compare with + testLabels. + + The first step towards doing this is to create a matrix of zeros with the + desired dimensions (1 x number_of_data_points). + + In predictionsTemp, the 3 dimensions for each data point correspond to the + probabilities of belonging to the three possible classes. + */ + arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); + + // Find index of max prediction for each data point and store in "prediction" + for (size_t i = 0; i < predictionTemp.n_cols; ++i) + { + // we add 1 to the max index, so that it matches the actual test labels. + prediction(i) = arma::as_scalar(arma::find( + arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; + } + + /* + Compute the error between predictions and testLabels, + now that we have the desired predictions. + */ + size_t correct = arma::accu(prediction == testLabels); + double classificationError = 1 - double(correct) / testData.n_cols; + + // Print out the classification error for the testing dataset. + std::cout << "Classification Error for the Test set: " << classificationError << std::endl; + return 0; } @endcode -Now, the matrix assignments holds the classification of each point in the -dataset. +Now, the matrix prediction holds the classification of each point in the +dataset. Subsequently, we find the classification error by comparing it +with testLabels. In the next example, we create simple noisy sine sequences, which are trained later on, using the RNN class in the `RNNModel()` method. @@ -328,7 +362,7 @@ implementation of a \c Forward() method. The interface looks like: @code template -void Forward(const arma::Mat&& input, arma::Mat&& output); +void Forward(const arma::Mat& input, arma::Mat& output); @endcode The method should calculate the output of the layer given the input matrix and @@ -339,9 +373,9 @@ through f: @code template -void Backward(const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g); +void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); @endcode Finally, if the layer is differentiable, the layer must also implement @@ -349,9 +383,9 @@ a Gradient() method: @code template -void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); +void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); @endcode The Gradient function should calculate the gradient with respect to the input @@ -434,21 +468,21 @@ API, so we must implement some additional functions. @code template -void Forward(const InputType&& input, OutputType&& output) +void Forward(const InputType& input, OutputType& output) { output = arma::ones(input.n_rows, input.n_cols); } template -void Backward(const InputType&& input, ErrorType&& gy, GradientType&& g) +void Backward(const InputType& input, const ErrorType& gy, GradientType& g) { g = arma::zeros(gy.n_rows, gy.n_cols) + gy; } template -void Gradient(const InputType&& input, - ErrorType&& error, - GradientType&& gradient) +void Gradient(const InputType& input, + ErrorType& error, + GradientType& gradient) { gradient = arma::zeros(input.n_rows, input.n_cols) * error; } diff --git a/doc/tutorials/image/image.txt b/doc/tutorials/image/image.txt new file mode 100644 index 0000000000..b64a86c24a --- /dev/null +++ b/doc/tutorials/image/image.txt @@ -0,0 +1,188 @@ +/*! +@file image.txt +@author Mehul Kumar Nirala +@brief Tutorial for how to load and save images in mlpack. + +@page imagetutorial Image Utilities tutorial + +@section intro_imagetut Introduction + +Image datasets are becoming increasingly popular in deep learning. + +mlpack's image saving/loading functionality is based on [stb/](https://github.com/nothings/stb). + +@section toc_imagetut Table of Contents + +This tutorial is split into the following sections: + + - \ref intro_imagetut + - \ref toc_imagetut + - \ref model_api_imagetut + - \ref imageinfo_api_imagetut + - \ref load_api_imagetut + - \ref save_api_imagetut + +@section model_api_imagetut Model API + +Image utilities supports loading and saving of images. + +It supports filetypes "jpg", "png", "tga","bmp", "psd", "gif", "hdr", "pic", "pnm" for loading and "jpg", "png", "tga", "bmp", "hdr" for saving. + +The datatype associated is unsigned char to support RGB values in the range 1-255. To feed data into the network typecast of `arma::Mat` may be required. Images are stored in matrix as (width * height * channels, NumberOfImages). Therefore imageMatrix.col(0) would be the first image if images are loaded in imageMatrix. + +@section imageinfo_api_imagetut ImageInfo + +ImageInfo class contains the metadata of the images. +@code + /** + * Instantiate the ImageInfo object with the image width, height, channels. + * + * @param width Image width. + * @param height Image height. + * @param channels number of channels in the image. + */ + ImageInfo(const size_t width, + const size_t height, + const size_t channels); +@endcode +Other public memebers include: + - quality Compression of the image if saved as jpg (0-100). + +@section load_api_imagetut Load + + +Standalone loading of images. +@code + /** + * Load the image file into the given matrix. + * + * @param filename Name of the image file. + * @param matrix Matrix to load the image into. + * @param info An object of ImageInfo class. + * @param fatal If an error should be reported as fatal (default false). + * @param transpose If true, flips the image, same as transposing the + * matrix after loading. + * @return Boolean value indicating success or failure of load. + */ + template + bool Load(const std::string& filename, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +@endcode + +Loading a test image. It also fills up the ImageInfo class object. +@code +data::ImageInfo info; +data::Load("test_image.png", matrix, info, false, true); +@endcode + +ImageInfo requires height, width, number of channels of the image. + +@code +size_t height = 64, width = 64, channels = 1; +data::ImageInfo info(width, height, channels); +@endcode + +More than one image can be loaded into the same matrix. + +Loading multiple images: + +@code + /** + * Load the image file into the given matrix. + * + * @param files A vector consisting of filenames. + * @param matrix Matrix to save the image from. + * @param info An object of ImageInfo class. + * @param fatal If an error should be reported as fatal (default false). + * @param transpose If true, flips the image, same as transposing the + * matrix after loading. + * @return Boolean value indicating success or failure of load. + */ + template + bool Load(const std::vector& files, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +@endcode + +@code + data::ImageInfo info; + std::vector> files{"test_image1.bmp","test_image2.bmp"}; + data::load(files, matrix, info, false, true); +@endcode + +@section save_api_imagetut Save + +Save images expects a matrix of type unsigned char in the form (width * height * channels, NumberOfImages). +Just like load it can be used to save one image or multiple images. Besides image data it also expects the shape of the image as input (width, height, channels). + +Saving one image: + +@code + /** + * Save the image file from the given matrix. + * + * @param filename Name of the image file. + * @param matrix Matrix to save the image from. + * @param info An object of ImageInfo class. + * @param fatal If an error should be reported as fatal (default false). + * @param transpose If true, flips the image, same as transposing the + * matrix after loading. + * @return Boolean value indicating success or failure of load. + */ + template + bool Save(const std::string& filename, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +@endcode + +@code + data::ImageInfo info; + info.width = info.height = 25; + info.channels = 3; + info.quality = 90; + data::Save("test_image.bmp", matrix, info, false, true); +@endcode + +If the matrix contains more than one image, only the first one is saved. + +Saving multiple images: + +@code + /** + * Save the image file from the given matrix. + * + * @param files A vector consisting of filenames. + * @param matrix Matrix to save the image from. + * @param info An object of ImageInfo class. + * @param fatal If an error should be reported as fatal (default false). + * @param transpose If true, Flips the image, same as transposing the + * matrix after loading. + * @return Boolean value indicating success or failure of load. + */ + template + bool Save(const std::vector& files, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +@endcode + +@code + data::ImageInfo info; + info.width = info.height = 25; + info.channels = 3; + info.quality = 90; + std::vector> files{"test_image1.bmp", "test_image2.bmp"}; + data::Save(files, matrix, info, false, true); +@endcode + +Multiple images are saved according to the vector of filenames specified. + +*/ diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt new file mode 100644 index 0000000000..a91dc27671 --- /dev/null +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -0,0 +1,399 @@ +/*! +@file rl.txt +@author Sriram S K +@author Joel Joseph +@brief Tutorial for how to use the Reinforcement Learning module in mlpack. + +@page rltutorial Reinforcement Learning Tutorial + +@section intro_rltut Introduction + +Reinforcement Learning is one of the hottest topics right now, with +interest surging after DeepMind published their article on training +deep neural networks to play Atari games to great success. mlpack +implements a complete end-to-end framework for Reinforcement Learning, +featuring multiple environments, policies and methods. Of course, +custom environments and policies can be used and plugged into the +existing framework with no runtime overhead. + +mlpack implements typical benchmark environments (Acrobot, Mountain car etc.), +commonly used policies, replay methods and supports asynchronous +learning as well. In addition, it can [communicate](https://github.com/zoq/gym_tcp_api) +with the OpenAI Gym toolkit for more environments. + +@section toc_rltut Table of Contents + +This tutorial is split into the following sections: + + - \ref intro_rltut + - \ref toc_rltut + - \ref environment_rltut + - \ref agent_components_rltut + - \ref q_learning_rltut + - \ref async_learning_rltut + - \ref further_rltut + +@section environment_rltut Reinforcement Learning Environments + +mlpack implements a number of the most popular environments used for testing +RL agents and algorithms. These include the Cart Pole, Acrobot, Mountain Car +and their variations. Of course, as mentioned above, you can communicate with +OpenAI Gym for other environments, like the Atari video games. + +A key component of mlpack is its extensibility. It is a simple process to create +your own custom environments, specific to your needs, and use it with mlpack's +RL framework. All the environments implement a few specific methods and classes +which are used by the agents while learning. + +- \c State: The State class is a representation of the environment. For the CartPole, + this would involve storing the position, velocity, angle and angular velocity. + +- \c Action: It is an enum naming all the possible actions the agent can take in the + environment. Continuing with the CartPole example, the Action enum would simply + contain the two possible actions, backward and forward. + +- \c Sample: This method is perhaps the heart of the environment, providing rewards to + the agent depending on the state and the action taken, and updates the state based on + the action taken as well. + +Of course, your custom environment will most likely make use of a number of helper methods, depending +on your application, such as the \c Dsdt method in the \c Acrobot environment, used in the \c RK4 +iterative method (also another helper method) to estimate the next state. + +@section agent_components_rltut Components of an RL Agent + +A Reinforcement Learning agent, in general, takes actions in an environment in order +to maximize a cumulative reward. To that end, it requires a way to choose actions (\b policy) +and a way to sample previous experiences (\b replay). + +An example of a simple policy would be an epsilon-greedy policy. Using such a policy, the agent +will choose actions greedily with some probability epsilon. This probability is slowly decreased +over time, balancing the line between exploration and exploitation. + +Similarly, an example of a simple replay would be a random replay. At each time step, the +interactions between the agent and the environment are saved to a memory buffer and previous +experiences are sampled from the buffer to train the agent. + +Instantiating the components of an agent can be easily done by passing the Environment as +a templated argument and the parameters of the policy/replay to the constructor. + +To create a Greedy Policy and Prioritized Replay for the CartPole environment, we would do the +following: + +@code +GreedyPolicy policy(1.0, 1000, 0.1); +PrioritizedReplay replayMethod(10, 10000, 0.6); +@endcode + +The arguments to `policy` are the initial epsilon values, the interval of decrease in its value +and the value at which epsilon bottoms out and won't be reduced further. The arguments to +`replayMethod` are size of the batch returned, the number of examples stored in memory, and the +degree of prioritization. + +In addition to the above components, an RL agent requires many hyperparameters to be tuned during + it's training period. These parameters include everything from the discount rate of the future +reward to whether Double Q-learning should be used or not. The `TrainingConfig` class can be +instantiated and configured as follows: + +@code + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.9; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 200; +@endcode + +The object `config` describes an RL agent, using a step size of 0.01 for the optimization process, +a discount factor of 0.9, sync interval of 200 episodes. This agent only starts learning after storing +100 exploration steps, has a step limit of 200, and does not utilize double q-learning. + +In this way, we can easily configure an RL agent with the desired hyperparameters. + +@section q_learning_rltut Q-Learning in mlpack + +Here, we demonstrate Q-Learning in mlpack through the use of a simple example, the training of a Q-Learning +agent on the CartPole environment. The code has been broken into chunks for easy understanding. + +@code +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace mlpack; +using namespace mlpack::ann; +using namespace ens; +using namespace mlpack::rl; +@endcode + +We include all the necessary components of our toy example and declare namespaces for convenience. + +@code +int main() +{ + // Set up the network. + FFN, GaussianInitialization> model(MeanSquaredError<>(), + GaussianInitialization(0, 0.001)); + model.Add>(4, 128); + model.Add>(); + model.Add>(128, 128); + model.Add>(); + model.Add>(128, 2); + +@endcode + +The first step in setting our Q-learning agent is to setup the network for it to use. Here, +we use mlpack's ann module to setup a simple FFN network, consisting of a single hidden layer. + +@note +The network constructed here has an input shape of 4 and output shape of 2. This corresponds to +the structure of the CartPole environment, where each state is represented as a column vector with +4 data members (position, velocity, angle, angular velocity). Similarly, the output shape is represented +by the number of possible actions, which in this case, is only 2 (foward and backward). + +The next step would be to setup the other components of the Q-learning agent, namely its policy, replay +method and hyperparameters. + +@code + // Set up the policy and replay method. + GreedyPolicy policy(1.0, 1000, 0.1, 0.99); + RandomReplay replayMethod(10, 10000); + + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.9; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 200; +@endcode + +And now, we get to the heart of the program, declaring a Q-Learning agent. + +@code + QLearning + agent(std::move(config), std::move(model), std::move(policy), + std::move(replayMethod)); +@endcode + +Here, we call the `QLearning` constructor, passing in the type of environment, +network, updater, policy and replay. We use `decltype(var)` as a shorthand for +the variable, saving us the trouble of copying the lengthy templated type. + +Similarly, `std::move` is called for convenience, moving the components instead of +duplicating them and copying them over. + +We have our Q-Learning agent `agent` ready to be trained on the Cart Pole environment. + +@code + arma::running_stat averageReturn; + size_t episodes = 0; + bool converged = true; + while (true) + { + double episodeReturn = agent.Episode(); + averageReturn(episodeReturn); + episodes += 1; + + if (episodes > 1000) + { + std::cout << "Cart Pole with DQN failed." << std::endl; + converged = false; + break; + } + + /** + * Reaching running average return 35 is enough to show it works. + */ + std::cout << "Average return: " << averageReturn.mean() + << " Episode return: " << episodeReturn << std::endl; + if (averageReturn.mean() > 35) + break; + } + if (converged) + std::cout << "Hooray! Q-Learning agent successfully trained" << std::endl; + + return 0; +} +@endcode + +We set up a loop to train the agent. The exit condition is determined by the average +reward which can be computed with `arma::running_stat`. It is used for storing running +statistics of scalars, which in this case is the reward signal. The agent can be said +to have converged when the average return reaches a predetermined value (i.e. > 35). + +Conversely, if the average return does not go beyond that amount even after a thousand +episodes, we can conclude that the agent will not converge and exit the training loop. + +@section async_learning_rltut + +In 2016, Researchers at Deepmind and University of Montreal published their paper +"Asynchronous Methods for Deep Reinforcement Learning". In it they described asynchronous +variants of four standard reinforcement learning algorithms: + - One-Step SARSA + - One-Step Q-Learning + - N-Step Q-Learning + - Advantage Actor-Critic(A3C) + +Online RL algorithms and Deep Neural Networks make an unstable combination because of the +non-stationary and correlated nature of online updates. Although this is solved by Experience Replay, +it has several drawbacks: it uses more memory and computation per real interaction; and it requires +off-policy learning algorithms. + +Asynchronous methods, instead of experience replay, asynchronously executes multiple agents +in parallel, on multiple instances of the environment, which solves all the above problems. + +Here, we demonstrate Asynchronous Learning methods in mlpack through the training of an async +agent. Asynchronous learning involves training several agents simultaneously. Here, each of the +agents are referred to as "workers". Currently mlpack has One-Step Q-Learning worker, N-Step +Q-Learning worker and One-Step SARSA worker. + +Let's examine the sample code in chunks. + +Apart from the includes used for the q-learning example, two more have to be included: + +@code +#include +#include +@endcode + +Here we don't use experience replay, and instead of a single policy, we use three different +policies, each corresponding to its worker. Number of workers created, depends on the number of +policies given in the Aggregated Policy. The column vector contains the probability distribution +for each child policy. We should make sure its size is same as the number of policies and the sum +of its elements is equal to 1. + +@code +AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), + GreedyPolicy(0.7, 5000, 0.01), + GreedyPolicy(0.7, 5000, 0.5)}, + arma::colvec("0.4 0.3 0.3")); +@endcode + +Now, we will create the "OneStepQLearning" agent. We could have used "NStepQLearning" or "OneStepSarsa" +here according to our requirement. + +@code +OneStepQLearning + agent(std::move(config), std::move(model), std::move(policy)); +@endcode + +Here, unlike the Q-Learning example, instead of the entire while loop, we use the Train method of the Asynchronous +Learning class inside a for loop. 100 training episodes will take around 50 seconds. + +@code +for (int i = 0; i < 100; i++) +{ + agent.Train(measure); +} +@endcode + +What is "measure" here? It is a lambda function which returns a boolean value (indicating the end of training) +and accepts the episode return (total reward of a deterministic test episode) as parameter. +So, let's create that. + +@code +arma::vec returns(20, arma::fill::zeros); +size_t position = 0; +size_t episode = 0; + +auto measure = [&returns, &position, &episode](double episodeReturn) +{ + if(episode > 10000) return true; + + returns[position++] = episodeReturn; + position = position % returns.n_elem; + episode++; + + std::cout << "Episode No.: " << episode + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << endl; +}; +@endcode + +This will train three different agents on three CPU threads asynchronously and use this data to update the +action value estimate. +Voila, thats all there is to it. + +Here is the full code to try this right away: + +@code +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace mlpack; +using namespace mlpack::ann; +using namespace mlpack::rl; +int main() +{ + // Set up the network. + FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); + model.Add>(4, 128); + model.Add>(); + model.Add>(128, 128); + model.Add>(); + model.Add>(128, 2); + + AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), + GreedyPolicy(0.7, 5000, 0.01), + GreedyPolicy(0.7, 5000, 0.5)}, + arma::colvec("0.4 0.3 0.3")); + + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.9; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 200; + + OneStepQLearning + agent(std::move(config), std::move(model), std::move(policy)); + + arma::vec returns(20, arma::fill::zeros); + size_t position = 0; + size_t episode = 0; + + auto measure = [&returns, &position, &episode](double episodeReturn) + { + if(episode > 10000) return true; + + returns[position++] = episodeReturn; + position = position % returns.n_elem; + episode++; + + std::cout << "Episode No.: " << episode + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << endl; + }; + + for (int i = 0; i < 100; i++) + { + agent.Train(measure); + } +} +@endcode + +@section further_rltut Further documentation + +For further documentation on the rl classes, consult the \ref mlpack::rl +"complete API documentation". + +*/ diff --git a/src/mlpack/bindings/julia/CMakeLists.txt b/src/mlpack/bindings/julia/CMakeLists.txt index d3249e96ef..461d7b7a22 100644 --- a/src/mlpack/bindings/julia/CMakeLists.txt +++ b/src/mlpack/bindings/julia/CMakeLists.txt @@ -18,10 +18,6 @@ if (BUILD_JULIA_BINDINGS) add_custom_command(TARGET julia PRE_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/src/) - add_custom_command(TARGET julia PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_SOURCE_DIR}/mlpack/REQUIRE - ${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/REQUIRE) add_library(mlpack_julia_util julia_util.h @@ -52,12 +48,16 @@ if (BUILD_JULIA_BINDINGS) configure_file(${CMAKE_SOURCE_DIR}/src/mlpack/bindings/julia/mlpack/cli.jl.in ${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/src/cli.jl) - # Create the empty mlpack.jl file that we will fill with includes. - file(WRITE - "${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/src/mlpack.jl" - "module mlpack\n\n" - "module util\n\n" - "include(\"cli.jl\")\n") + # Create the empty mlpack.jl file that we will fill with includes using the + # exsiting template. Unfortunately COPY doesn't let us change the extension + # so we need a follow-up RENAME command. + file(COPY + "${CMAKE_CURRENT_SOURCE_DIR}/mlpack/mlpack.jl.in" + DESTINATION + "${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/src/") + file(RENAME + "${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/src/mlpack.jl.in" + "${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/src/mlpack.jl") file(WRITE "${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/src/functions.jl" @@ -150,10 +150,10 @@ if (BUILD_JULIA_BINDINGS) # Append the code to define the function in the module. file(APPEND "${CMAKE_BINARY_DIR}/src/mlpack/bindings/julia/mlpack/src/functions.jl" - "${name} = util.${name}\n") + "${name} = _Internal.${name}\n") endif () endmacro () -if (BUILD_TESTS) +if (BUILD_TESTS AND BUILD_JULIA_BINDINGS) add_subdirectory(tests) endif () diff --git a/src/mlpack/bindings/julia/default_param_impl.hpp b/src/mlpack/bindings/julia/default_param_impl.hpp index 6a25fb494f..47f1bd7ab1 100644 --- a/src/mlpack/bindings/julia/default_param_impl.hpp +++ b/src/mlpack/bindings/julia/default_param_impl.hpp @@ -116,11 +116,11 @@ std::string DefaultParamImpl( else if (std::is_same>::value || std::is_same>::value) { - return "Int64[]"; + return "Int[]"; } else if (std::is_same>::value) { - return "zeros(Int64, 0, 0)"; + return "zeros(Int, 0, 0)"; } else { diff --git a/src/mlpack/bindings/julia/get_julia_type.hpp b/src/mlpack/bindings/julia/get_julia_type.hpp index 9ff62e4cb9..096c420ab9 100644 --- a/src/mlpack/bindings/julia/get_julia_type.hpp +++ b/src/mlpack/bindings/julia/get_julia_type.hpp @@ -101,9 +101,9 @@ inline std::string GetJuliaType( const typename std::enable_if::value>::type* = 0) { // size_t matrices are special: we want to represent them in Julia as - // Array{Int64, X} not UInt64 because Julia displays UInt64s strangely. + // Array{Int, X} not UInt because Julia displays UInts strangely. if (std::is_same::value) - return std::string("Array{Int64, ") + (T::is_col || T::is_row ? "1" : "2") + return std::string("Array{Int, ") + (T::is_col || T::is_row ? "1" : "2") + "}"; else return "Array{" + GetJuliaType() + ", " diff --git a/src/mlpack/bindings/julia/get_printable_type_impl.hpp b/src/mlpack/bindings/julia/get_printable_type_impl.hpp index c46e75b365..524fe62a0b 100644 --- a/src/mlpack/bindings/julia/get_printable_type_impl.hpp +++ b/src/mlpack/bindings/julia/get_printable_type_impl.hpp @@ -47,7 +47,7 @@ std::string GetPrintableType( const typename std::enable_if::value>::type*) { if (std::is_same>::value) - return "Array{Int64, 1}"; + return "Array{Int, 1}"; else if (std::is_same>::value) return "Array{String, 1}"; else @@ -65,15 +65,15 @@ std::string GetPrintableType( if (std::is_same::value) return "Float64 matrix-like"; else if (std::is_same>::value) - return "Int64 matrix-like"; + return "Int matrix-like"; else if (std::is_same::value) return "Float64 vector-like"; else if (std::is_same>::value) - return "Int64 vector-like"; + return "Int vector-like"; else if (std::is_same::value) return "Float64 vector-like"; else if (std::is_same>::value) - return "Int64 vector-like"; + return "Int vector-like"; else throw std::invalid_argument("unknown Armadillo type " + data.cppType); } diff --git a/src/mlpack/bindings/julia/julia_util.cpp b/src/mlpack/bindings/julia/julia_util.cpp index 8db14179a3..faa639956f 100644 --- a/src/mlpack/bindings/julia/julia_util.cpp +++ b/src/mlpack/bindings/julia/julia_util.cpp @@ -7,6 +7,7 @@ #include #include #include +#include using namespace mlpack; @@ -82,13 +83,14 @@ void CLI_SetParamVectorStrStr(const char* paramName, * Call CLI::SetParam>(). */ void CLI_SetParamVectorInt(const char* paramName, - uint64_t* ints, + int* ints, const size_t length) { // Create a std::vector object; unfortunately this requires copying the // vector elements. - std::vector vec(length); - for (size_t i = 0; i < (size_t) length; ++i) + std::vector vec; + vec.resize(length); + for (size_t i = 0; i < length; ++i) vec[i] = ints[i]; CLI::GetParam>(paramName) = std::move(vec); @@ -105,7 +107,7 @@ void CLI_SetParamMat(const char* paramName, const bool pointsAsRows) { // Create the matrix as an alias. - arma::mat m(memptr, rows, cols, false, true); + arma::mat m(memptr, arma::uword(rows), arma::uword(cols), false, true); CLI::GetParam(paramName) = pointsAsRows ? m.t() : std::move(m); CLI::SetPassed(paramName); } @@ -120,7 +122,8 @@ void CLI_SetParamUMat(const char* paramName, const bool pointsAsRows) { // Create the matrix as an alias. - arma::Mat m(memptr, rows, cols, false, true); + arma::Mat m(memptr, arma::uword(rows), arma::uword(cols), false, + true); CLI::GetParam>(paramName) = pointsAsRows ? m.t() : std::move(m); CLI::SetPassed(paramName); @@ -133,7 +136,7 @@ void CLI_SetParamRow(const char* paramName, double* memptr, const size_t cols) { - arma::rowvec m(memptr, cols, false, true); + arma::rowvec m(memptr, arma::uword(cols), false, true); CLI::GetParam(paramName) = std::move(m); CLI::SetPassed(paramName); } @@ -145,7 +148,7 @@ void CLI_SetParamURow(const char* paramName, size_t* memptr, const size_t cols) { - arma::Row m(memptr, cols, false, true); + arma::Row m(memptr, arma::uword(cols), false, true); CLI::GetParam>(paramName) = std::move(m); CLI::SetPassed(paramName); } @@ -157,7 +160,7 @@ void CLI_SetParamCol(const char* paramName, double* memptr, const size_t rows) { - arma::vec m(memptr, rows, false, true); + arma::vec m(memptr, arma::uword(rows), false, true); CLI::GetParam(paramName) = std::move(m); CLI::SetPassed(paramName); } @@ -166,10 +169,10 @@ void CLI_SetParamCol(const char* paramName, * Call CLI::SetParam>(). */ void CLI_SetParamUCol(const char* paramName, - size_t* memptr, - const size_t rows) + size_t* memptr, + const size_t rows) { - arma::Col m(memptr, rows, false, true); + arma::Col m(memptr, arma::uword(rows), false, true); CLI::GetParam>(paramName) = std::move(m); CLI::SetPassed(paramName); } @@ -191,7 +194,7 @@ void CLI_SetParamMatWithInfo(const char* paramName, data::Datatype::numeric; } - arma::mat m(memptr, rows, cols, false, true); + arma::mat m(memptr, arma::uword(rows), arma::uword(cols), false, true); std::get<0>(CLI::GetParam>( paramName)) = std::move(d); std::get<1>(CLI::GetParam>( @@ -243,7 +246,7 @@ size_t CLI_GetParamVectorStrLen(const char* paramName) /** * Call CLI::GetParam>() and get the i'th string. */ -const char* CLI_GetParamVectorStrStr(const char* paramName, const int i) +const char* CLI_GetParamVectorStrStr(const char* paramName, const size_t i) { return CLI::GetParam>(paramName)[i].c_str(); } @@ -261,10 +264,10 @@ size_t CLI_GetParamVectorIntLen(const char* paramName) * The vector will be created in-place and it is expected that the calling * function will take ownership. */ -uint64_t* CLI_GetParamVectorIntPtr(const char* paramName) +int* CLI_GetParamVectorIntPtr(const char* paramName) { const size_t size = CLI::GetParam>(paramName).size(); - uint64_t* ints = new uint64_t[size]; + int* ints = new int[size]; for (size_t i = 0; i < size; ++i) ints[i] = CLI::GetParam>(paramName)[i]; @@ -335,15 +338,17 @@ size_t CLI_GetParamUMatCols(const char* paramName) */ size_t* CLI_GetParamUMat(const char* paramName) { + arma::Mat& mat = CLI::GetParam>(paramName); + // Are we using preallocated memory? If so we have to handle this more // carefully. - arma::Mat& mat = CLI::GetParam>(paramName); if (mat.n_elem <= arma::arma_config::mat_prealloc) { // Copy the memory to something that we can give back to Julia. size_t* newMem = new size_t[mat.n_elem]; arma::arrayops::copy(newMem, mat.mem, mat.n_elem); - return newMem; // We believe Julia will free it. Hopefully we are right. + // We believe Julia will free it. Hopefully we are right. + return newMem; } else { @@ -399,15 +404,17 @@ size_t CLI_GetParamUColRows(const char* paramName) */ size_t* CLI_GetParamUCol(const char* paramName) { + arma::Col& vec = CLI::GetParam>(paramName); + // Are we using preallocated memory? If so we have to handle this more // carefully. - arma::Col& vec = CLI::GetParam>(paramName); if (vec.n_elem <= arma::arma_config::mat_prealloc) { // Copy the memory to something we can give back to Julia. size_t* newMem = new size_t[vec.n_elem]; arma::arrayops::copy(newMem, vec.mem, vec.n_elem); - return newMem; // We believe Julia will free it. Hopefully we are right. + // We believe Julia will free it. Hopefully we are right. + return newMem; } else { @@ -463,9 +470,10 @@ size_t CLI_GetParamURowCols(const char* paramName) */ size_t* CLI_GetParamURow(const char* paramName) { + arma::Row& vec = CLI::GetParam>(paramName); + // Are we using preallocated memory? If so we have to handle this more // carefully. - arma::Row& vec = CLI::GetParam>(paramName); if (vec.n_elem <= arma::arma_config::mat_prealloc) { // Copy the memory to something we can give back to Julia. diff --git a/src/mlpack/bindings/julia/julia_util.h b/src/mlpack/bindings/julia/julia_util.h index 706d363919..06ec97ad0a 100644 --- a/src/mlpack/bindings/julia/julia_util.h +++ b/src/mlpack/bindings/julia/julia_util.h @@ -61,7 +61,7 @@ void CLI_SetParamVectorStrStr(const char* paramName, * Call CLI::SetParam>(). */ void CLI_SetParamVectorInt(const char* paramName, - uint64_t* ints, + int* ints, const size_t length); /** @@ -149,7 +149,7 @@ size_t CLI_GetParamVectorStrLen(const char* paramName); /** * Call CLI::GetParam>() and get the i'th string. */ -const char* CLI_GetParamVectorStrStr(const char* paramName, const int i); +const char* CLI_GetParamVectorStrStr(const char* paramName, const size_t i); /** * Call CLI::GetParam>() and get the length of the vector. @@ -161,7 +161,7 @@ size_t CLI_GetParamVectorIntLen(const char* paramName); * The vector will be created in-place and it is expected that the calling * function will take ownership. */ -uint64_t* CLI_GetParamVectorIntPtr(const char* paramName); +int* CLI_GetParamVectorIntPtr(const char* paramName); /** * Get the number of rows in a matrix parameter. diff --git a/src/mlpack/bindings/julia/mlpack/REQUIRE b/src/mlpack/bindings/julia/mlpack/REQUIRE deleted file mode 100644 index aef1ca2bd9..0000000000 --- a/src/mlpack/bindings/julia/mlpack/REQUIRE +++ /dev/null @@ -1 +0,0 @@ -julia 0.7.0 diff --git a/src/mlpack/bindings/julia/mlpack/cli.jl.in b/src/mlpack/bindings/julia/mlpack/cli.jl.in index 687f532a2e..b5f7c2c0a6 100644 --- a/src/mlpack/bindings/julia/mlpack/cli.jl.in +++ b/src/mlpack/bindings/julia/mlpack/cli.jl.in @@ -66,8 +66,8 @@ function CLIRestoreSettings(programName::String) end function CLISetParam(paramName::String, paramValue::Int) - ccall((:CLI_SetParamInt, library), Nothing, (Cstring, Int), paramName, - paramValue); + ccall((:CLI_SetParamInt, library), Nothing, (Cstring, Cint), paramName, + Cint(paramValue)); end function CLISetParam(paramName::String, paramValue::Float64) @@ -89,15 +89,15 @@ function CLISetParamMat(paramName::String, paramValue, pointsAsRows::Bool) paramMat = to_matrix(paramValue, Float64) - ccall((:CLI_SetParamMat, library), Nothing, (Cstring, Ptr{Float64}, UInt64, - UInt64, Bool), paramName, Base.pointer(paramMat), size(paramMat, 1), + ccall((:CLI_SetParamMat, library), Nothing, (Cstring, Ptr{Float64}, Csize_t, + Csize_t, Bool), paramName, Base.pointer(paramMat), size(paramMat, 1), size(paramMat, 2), pointsAsRows); end function CLISetParamUMat(paramName::String, paramValue, pointsAsRows::Bool) - paramMat = to_matrix(paramValue, Int64) + paramMat = to_matrix(paramValue, Int) # Sanity check. if minimum(paramMat) <= 0 @@ -105,9 +105,9 @@ function CLISetParamUMat(paramName::String, "Must be 1 or greater.")) end - m = convert(Array{UInt64, 2}, paramMat .- 1) - ccall((:CLI_SetParamUMat, library), Nothing, (Cstring, Ptr{UInt64}, UInt64, - UInt64, Bool), paramName, Base.pointer(m), size(paramValue, 1), + m = convert(Array{Csize_t, 2}, paramMat .- 1) + ccall((:CLI_SetParamUMat, library), Nothing, (Cstring, Ptr{Csize_t}, Csize_t, + Csize_t, Bool), paramName, Base.pointer(m), size(paramValue, 1), size(paramValue, 2), pointsAsRows); end @@ -117,25 +117,26 @@ function CLISetParam(paramName::String, # sequentially. I am not sure if this is fully necessary but I have some # reservations about Julia's support for passing arrays of strings correctly # as a const char**. - ccall((:CLI_SetParamVectorStrLen, library), Nothing, (Cstring, UInt64), + ccall((:CLI_SetParamVectorStrLen, library), Nothing, (Cstring, Csize_t), paramName, size(vector, 1)); for i in 1:size(vector, 1) ccall((:CLI_SetParamVectorStrStr, library), Nothing, (Cstring, Cstring, - UInt64), paramName, vector[i], i .- 1); + Csize_t), paramName, vector[i], i .- 1); end end function CLISetParam(paramName::String, - vector::Vector{Int64}) - ccall((:CLI_SetParamVectorInt, library), Nothing, (Cstring, Ptr{Int64}, - Int64), paramName, Base.pointer(vector), size(vector, 1)); + vector::Vector{Int}) + cint_vec = convert(Vector{Cint}, vector) + ccall((:CLI_SetParamVectorInt, library), Nothing, (Cstring, Ptr{Cint}, + Csize_t), paramName, Base.pointer(cint_vec), size(cint_vec, 1)); end function CLISetParam(paramName::String, matWithInfo::Tuple{Array{Bool, 1}, Array{Float64, 2}}, pointsAsRows::Bool) ccall((:CLI_SetParamMatWithInfo, library), Nothing, (Cstring, Ptr{Bool}, - Ptr{Float64}, Int64, Int64, Bool), paramName, + Ptr{Float64}, Int, Int, Bool), paramName, Base.pointer(matWithInfo[1]), Base.pointer(matWithInfo[2]), size(matWithInfo[2], 1), size(matWithInfo[2], 2), pointsAsRows); end @@ -143,44 +144,44 @@ end function CLISetParamRow(paramName::String, paramValue) paramVec = to_vector(paramValue, Float64) - ccall((:CLI_SetParamRow, library), Nothing, (Cstring, Ptr{Float64}, UInt64), + ccall((:CLI_SetParamRow, library), Nothing, (Cstring, Ptr{Float64}, Csize_t), paramName, Base.pointer(paramVec), size(paramVec, 1)); end function CLISetParamCol(paramName::String, paramValue) paramVec = to_vector(paramValue, Float64) - ccall((:CLI_SetParamCol, library), Nothing, (Cstring, Ptr{Float64}, UInt64), + ccall((:CLI_SetParamCol, library), Nothing, (Cstring, Ptr{Float64}, Csize_t), paramName, Base.pointer(paramVec), size(paramVec, 1)); end function CLISetParamURow(paramName::String, paramValue) - paramVec = to_vector(paramValue, Int64) + paramVec = to_vector(paramValue, Int) # Sanity check. if minimum(paramVec) <= 0 throw(DomainError("Input $(paramName) cannot have 0 or negative values! " * "Must be 1 or greater.")) end - m = convert(Array{UInt64, 1}, paramVec .- 1) + m = convert(Array{Csize_t, 1}, paramVec .- 1) - ccall((:CLI_SetParamURow, library), Nothing, (Cstring, Ptr{UInt64}, UInt64), + ccall((:CLI_SetParamURow, library), Nothing, (Cstring, Ptr{Csize_t}, Csize_t), paramName, Base.pointer(m), size(paramValue, 1)); end function CLISetParamUCol(paramName::String, paramValue) - paramVec = to_vector(paramValue, Int64) + paramVec = to_vector(paramValue, Int) # Sanity check. if minimum(paramVec) <= 0 throw(DomainError("Input $(paramName) cannot have 0 or negative values! " * "Must be 1 or greater.")) end - m = convert(Array{UInt64, 1}, paramValue .- 1) + m = convert(Array{Csize_t, 1}, paramValue .- 1) - ccall((:CLI_SetParamUCol, library), Nothing, (Cstring, Ptr{UInt64}, UInt64), + ccall((:CLI_SetParamUCol, library), Nothing, (Cstring, Ptr{Csize_t}, Csize_t), paramName, Base.pointer(m), size(paramValue, 1)); end @@ -189,7 +190,7 @@ function CLIGetParamBool(paramName::String) end function CLIGetParamInt(paramName::String) - return ccall((:CLI_GetParamInt, library), Int64, (Cstring,), paramName) + return Int(ccall((:CLI_GetParamInt, library), Cint, (Cstring,), paramName)) end function CLIGetParamDouble(paramName::String) @@ -201,15 +202,15 @@ function CLIGetParamString(paramName::String) end function CLIGetParamVectorStr(paramName::String) - local size::UInt64 + local size::Csize_t local ptr::Ptr{String} # Get the size of the vector, then each element. - size = ccall((:CLI_GetParamVectorStrLen, library), UInt64, (Cstring,), + size = ccall((:CLI_GetParamVectorStrLen, library), Csize_t, (Cstring,), paramName); out = Array{String, 1}() for i = 1:size - s = ccall((:CLI_GetParamVectorStrStr, library), Cstring, (Cstring, UInt64), + s = ccall((:CLI_GetParamVectorStrStr, library), Cstring, (Cstring, Csize_t), paramName, i .- 1) push!(out, Base.unsafe_string(s)) end @@ -218,28 +219,29 @@ function CLIGetParamVectorStr(paramName::String) end function CLIGetParamVectorInt(paramName::String) - local size::UInt64 - local ptr::Ptr{Int64} + local size::Csize_t + local ptr::Ptr{Cint} # Get the size of the vector, then the pointer to it. We will own the # pointer. - size = ccall((:CLI_GetParamVectorIntLen, library), UInt64, (Cstring,), + size = ccall((:CLI_GetParamVectorIntLen, library), Csize_t, (Cstring,), paramName); - ptr = ccall((:CLI_GetParamVectorIntPtr, library), Ptr{Int64}, (Cstring,), + ptr = ccall((:CLI_GetParamVectorIntPtr, library), Ptr{Cint}, (Cstring,), paramName); - return Base.unsafe_wrap(Array{Int64, 1}, ptr, (size), own=true) + return convert(Array{Int, 1}, Base.unsafe_wrap(Array{Cint, 1}, ptr, (size), + own=true)) end function CLIGetParamMat(paramName::String, pointsAsRows::Bool) # Can we return different return types? For now let's restrict to a matrix to # make it easy... local ptr::Ptr{Float64} - local rows::UInt64, cols::UInt64; + local rows::Csize_t, cols::Csize_t; # I suppose it would be possible to do this all in one call, but this seems # easy enough. - rows = ccall((:CLI_GetParamMatRows, library), UInt64, (Cstring,), paramName); - cols = ccall((:CLI_GetParamMatCols, library), UInt64, (Cstring,), paramName); + rows = ccall((:CLI_GetParamMatRows, library), Csize_t, (Cstring,), paramName); + cols = ccall((:CLI_GetParamMatCols, library), Csize_t, (Cstring,), paramName); ptr = ccall((:CLI_GetParamMat, library), Ptr{Float64}, (Cstring,), paramName); if pointsAsRows @@ -255,30 +257,30 @@ end function CLIGetParamUMat(paramName::String, pointsAsRows::Bool) # Can we return different return types? For now let's restrict to a matrix to # make it easy... - local ptr::Ptr{UInt64} - local rows::UInt64, cols::UInt64; + local ptr::Ptr{Csize_t} + local rows::Csize_t, cols::Csize_t; # I suppose it would be possible to do this all in one call, but this seems # easy enough. - rows = ccall((:CLI_GetParamUMatRows, library), UInt64, (Cstring,), paramName); - cols = ccall((:CLI_GetParamUMatCols, library), UInt64, (Cstring,), paramName); - ptr = ccall((:CLI_GetParamUMat, library), Ptr{UInt64}, (Cstring,), paramName); + rows = ccall((:CLI_GetParamUMatRows, library), Csize_t, (Cstring,), paramName); + cols = ccall((:CLI_GetParamUMatCols, library), Csize_t, (Cstring,), paramName); + ptr = ccall((:CLI_GetParamUMat, library), Ptr{Csize_t}, (Cstring,), paramName); if pointsAsRows # In this case we have to transpose, unfortunately. - m = Base.unsafe_wrap(Array{UInt64, 2}, ptr, (rows, cols), own=true); - return convert(Array{Int64, 2}, m' .+ 1) # Add 1 because these are indexes. + m = Base.unsafe_wrap(Array{Csize_t, 2}, ptr, (rows, cols), own=true); + return convert(Array{Int, 2}, m' .+ 1) # Add 1 because these are indexes. else # Here no transpose is necessary. - m = Base.unsafe_wrap(Array{UInt64, 2}, ptr, (rows, cols), own=true); - return convert(Array{Int64, 2}, m .+ 1) + m = Base.unsafe_wrap(Array{Csize_t, 2}, ptr, (rows, cols), own=true); + return convert(Array{Int, 2}, m .+ 1) end end function CLIGetParamCol(paramName::String) local ptr::Ptr{Float64}; - local rows::UInt64; + local rows::Csize_t; - rows = ccall((:CLI_GetParamColRows, library), UInt64, (Cstring,), paramName); + rows = ccall((:CLI_GetParamColRows, library), Csize_t, (Cstring,), paramName); ptr = ccall((:CLI_GetParamCol, library), Ptr{Float64}, (Cstring,), paramName); return Base.unsafe_wrap(Array{Float64, 1}, ptr, rows, own=true); @@ -286,45 +288,45 @@ end function CLIGetParamRow(paramName::String) local ptr::Ptr{Float64}; - local cols::UInt64; + local cols::Csize_t; - cols = ccall((:CLI_GetParamRowCols, library), UInt64, (Cstring,), paramName); + cols = ccall((:CLI_GetParamRowCols, library), Csize_t, (Cstring,), paramName); ptr = ccall((:CLI_GetParamRow, library), Ptr{Float64}, (Cstring,), paramName); return Base.unsafe_wrap(Array{Float64, 1}, ptr, cols, own=true); end function CLIGetParamUCol(paramName::String) - local ptr::Ptr{UInt64}; - local rows::UInt64; + local ptr::Ptr{Csize_t}; + local rows::Csize_t; - rows = ccall((:CLI_GetParamUColRows, library), UInt64, (Cstring,), paramName); - ptr = ccall((:CLI_GetParamUCol, library), Ptr{UInt64}, (Cstring,), paramName); + rows = ccall((:CLI_GetParamUColRows, library), Csize_t, (Cstring,), paramName); + ptr = ccall((:CLI_GetParamUCol, library), Ptr{Csize_t}, (Cstring,), paramName); - m = Base.unsafe_wrap(Array{UInt64, 1}, ptr, rows, own=true); - return convert(Array{Int64, 1}, m .+ 1) + m = Base.unsafe_wrap(Array{Csize_t, 1}, ptr, rows, own=true); + return convert(Array{Int, 1}, m .+ 1) end function CLIGetParamURow(paramName::String) - local ptr::Ptr{UInt64}; - local cols::UInt64; + local ptr::Ptr{Csize_t}; + local cols::Csize_t; - cols = ccall((:CLI_GetParamURowCols, library), UInt64, (Cstring,), paramName); - ptr = ccall((:CLI_GetParamURow, library), Ptr{UInt64}, (Cstring,), paramName); + cols = ccall((:CLI_GetParamURowCols, library), Csize_t, (Cstring,), paramName); + ptr = ccall((:CLI_GetParamURow, library), Ptr{Csize_t}, (Cstring,), paramName); - m = Base.unsafe_wrap(Array{UInt64, 1}, ptr, cols, own=true); - return convert(Array{Int64, 1}, m .+ 1) + m = Base.unsafe_wrap(Array{Csize_t, 1}, ptr, cols, own=true); + return convert(Array{Int, 1}, m .+ 1) end function CLIGetParamMatWithInfo(paramName::String, pointsAsRows::Bool) local ptrBool::Ptr{Bool}; local ptrData::Ptr{Float64}; - local rows::UInt64; - local cols::UInt64; + local rows::Csize_t; + local cols::Csize_t; - rows = ccall((:CLI_GetParamMatWithInfoRows, library), UInt64, (Cstring,), + rows = ccall((:CLI_GetParamMatWithInfoRows, library), Csize_t, (Cstring,), paramName); - cols = ccall((:CLI_GetParamMatWithInfoCols, library), UInt64, (Cstring,), + cols = ccall((:CLI_GetParamMatWithInfoCols, library), Csize_t, (Cstring,), paramName); ptrBool = ccall((:CLI_GetParamMatWithInfoBoolPtr, library), Ptr{Bool}, (Cstring,), paramName); diff --git a/src/mlpack/bindings/julia/mlpack/mlpack.jl.in b/src/mlpack/bindings/julia/mlpack/mlpack.jl.in new file mode 100644 index 0000000000..b2368ecb75 --- /dev/null +++ b/src/mlpack/bindings/julia/mlpack/mlpack.jl.in @@ -0,0 +1,30 @@ +""" + mlpack + +mlpack is a fast, flexible machine learning library, written in C++, that aims +to provide fast, extensible implementations of cutting-edge machine learning +algorithms. This module provides those implementations as Julia functions. + +Each function inside the module performs a specific machine learning task. + +For complete documentation of these functions, including example usage, see the +mlpack website's documentation for the Julia bindings: + +https://www.mlpack.org/doc/stable/julia_documentation.html + +Each function also contains an equivalent docstring; the Julia REPL's help +functionality can be used to access the documentation that way. +""" +module mlpack + +""" + mlpack._Internal + +This module contains internal implementations details of mlpack. There +shouldn't be any need to go digging around in here if you're just using mlpack. +(But don't let this comment discourage you if you're just curious and poking +around!) +""" +module _Internal + +include("cli.jl") diff --git a/src/mlpack/bindings/julia/print_doc_functions_impl.hpp b/src/mlpack/bindings/julia/print_doc_functions_impl.hpp index c428b5986a..8fb2534f72 100644 --- a/src/mlpack/bindings/julia/print_doc_functions_impl.hpp +++ b/src/mlpack/bindings/julia/print_doc_functions_impl.hpp @@ -124,7 +124,7 @@ inline std::string CreateInputArguments(const std::string& paramName, d.cppType == "arma::Col") { oss << "julia> " << value << " = CSV.read(\"" << value - << ".csv\"; type=Int64)" << std::endl; + << ".csv\"; type=Int)" << std::endl; } } @@ -375,6 +375,9 @@ inline std::string ProgramCall(const std::string& programName, Args... args) { std::ostringstream oss; + // The code should appear in a Markdown code block. + oss << "```julia" << std::endl; + // Print any input argument definitions. The only input argument definitions // will be the definitions of matrices, which use the CSV.jl package, so we // should also include a `using CSV` in there too. @@ -401,6 +404,9 @@ inline std::string ProgramCall(const std::string& programName, Args... args) // Since `julia> ` is 8 characters, let's indent 12 otherwise it looks weird. oss << util::HyphenateString(ossCall.str(), 12); + // Close the Markdown code block. + oss << std::endl << "```"; + return oss.str(); } diff --git a/src/mlpack/bindings/julia/print_jl.cpp b/src/mlpack/bindings/julia/print_jl.cpp index 939b606386..7f4fb86a02 100644 --- a/src/mlpack/bindings/julia/print_jl.cpp +++ b/src/mlpack/bindings/julia/print_jl.cpp @@ -62,7 +62,7 @@ void PrintJL(const util::ProgramDoc& programInfo, cout << endl; // We need to include utility functions. - cout << "using mlpack.util.cli" << endl; + cout << "using mlpack._Internal.cli" << endl; cout << endl; // Make sure the libraries we need are accessible. diff --git a/src/mlpack/bindings/julia/print_type_doc_impl.hpp b/src/mlpack/bindings/julia/print_type_doc_impl.hpp index 9e81099bfd..e6522d36d2 100644 --- a/src/mlpack/bindings/julia/print_type_doc_impl.hpp +++ b/src/mlpack/bindings/julia/print_type_doc_impl.hpp @@ -109,15 +109,15 @@ std::string PrintTypeDoc( if (T::is_col || T::is_row) { return "A 1-d vector-like containing `Int` data (elements should be " - "greater than or equal to 0). Could be an `Array{Int64, 1}`, an " - "`Array{Int64, 2}` with one dimension of size 1, or anything " - "convertible to `Array{Int64, 1}`."; + "greater than or equal to 0). Could be an `Array{Int, 1}`, an " + "`Array{Int, 2}` with one dimension of size 1, or anything " + "convertible to `Array{Int, 1}`."; } else { return "A 2-d matrix-like containing `Int` data (elements should be " - "greater than or equal to 0). Could be an `Array{Int64, 2}` or a " - "`DataFrame` or anything convertible to an `Array{Int64, 2}`. It is " + "greater than or equal to 0). Could be an `Array{Int, 2}` or a " + "`DataFrame` or anything convertible to an `Array{Int, 2}`. It is " "expected that each row of the matrix corresponds to a data point, " "unless `points_are_rows` is set to `false` when calling mlpack " "bindings."; diff --git a/src/mlpack/bindings/julia/tests/runtests.jl b/src/mlpack/bindings/julia/tests/runtests.jl index 616a0c2804..7a859063a0 100644 --- a/src/mlpack/bindings/julia/tests/runtests.jl +++ b/src/mlpack/bindings/julia/tests/runtests.jl @@ -129,7 +129,7 @@ end # Same as TestMatrix but with an unsigned matrix. @testset "TestUMatrix" begin # Generate a random matrix of integers. - x = convert(Array{Int64, 2}, rand(1:500, (100, 5))) + x = convert(Array{Int, 2}, rand(1:500, (100, 5))) _, _, _, _, _, _, _, _, _, _, _, umatOut, _, _ = test_julia_binding(4.0, 12, "hello", @@ -138,7 +138,7 @@ end @test size(umatOut, 1) == 100 @test size(umatOut, 2) == 4 - @test typeof(umatOut[1, 1]) == Int64 + @test typeof(umatOut[1, 1]) == Int for i in [0, 1, 3] for j in 1:100 @test umatOut[j, i + 1] == x[j, i + 1] @@ -155,7 +155,7 @@ end # Same as TestMatrix but with an unsigned column major matrix. @testset "TestUMatrixColMajor" begin # Generate a random matrix of integers. - x = convert(Array{Int64, 2}, rand(1:500, (5, 100))) + x = convert(Array{Int, 2}, rand(1:500, (5, 100))) _, _, _, _, _, _, _, _, _, _, _, umatOut, _, _ = test_julia_binding(4.0, 12, "hello", @@ -164,7 +164,7 @@ end @test size(umatOut, 1) == 4 @test size(umatOut, 2) == 100 - @test typeof(umatOut[1, 1]) == Int64 + @test typeof(umatOut[1, 1]) == Int for i in 1:100 for j in [0, 1, 3] @test umatOut[j + 1, i] == x[j + 1, i] @@ -196,14 +196,14 @@ end # Test an unsigned column vector input parameter. @testset "TestUCol" begin - x = convert(Array{Int64, 1}, rand(1:500, 100)) + x = convert(Array{Int, 1}, rand(1:500, 100)) _, _, _, _, _, _, _, _, _, _, ucolOut, _, _, _ = test_julia_binding(4.0, 12, "hello", ucol_in=x) @test size(ucolOut, 1) == 100 - @test typeof(ucolOut) == Array{Int64, 1} + @test typeof(ucolOut) == Array{Int, 1} for i in 1:100 # Since we subtract one when we convert to C++, and then add one when we # convert back, we get a slightly different result here. @@ -228,14 +228,14 @@ end # Test an unsigned row vector input parameter. @testset "TestURow" begin - x = convert(Array{Int64, 1}, rand(1:500, 100)) + x = convert(Array{Int, 1}, rand(1:500, 100)) _, _, _, _, _, _, _, _, _, _, _, _, urowOut, _ = test_julia_binding(4.0, 12, "hello", urow_in=x) @test size(urowOut, 1) == 100 - @test typeof(urowOut) == Array{Int64, 1} + @test typeof(urowOut) == Array{Int, 1} for i in 1:100 # Since we subtract one when we convert to C++, and then add one when we # convert back, we get a slightly different result here. diff --git a/src/mlpack/bindings/markdown/print_doc_functions_impl.hpp b/src/mlpack/bindings/markdown/print_doc_functions_impl.hpp index 28e0fbea6d..0e9da65798 100644 --- a/src/mlpack/bindings/markdown/print_doc_functions_impl.hpp +++ b/src/mlpack/bindings/markdown/print_doc_functions_impl.hpp @@ -451,20 +451,21 @@ inline std::string PrintModel(const std::string& model) template std::string ProgramCall(const std::string& programName, Args... args) { - std::string s = "```"; + std::string s; if (BindingInfo::Language() == "cli") { - s += "bash\n"; + s += "```bash\n"; s += cli::ProgramCall(programName, args...); } else if (BindingInfo::Language() == "python") { - s += "python\n"; + s += "```python\n"; s += python::ProgramCall(programName, args...); } else if (BindingInfo::Language() == "julia") { - s += "julia\n"; + // Julia's ProgramCall() with a set of arguments will automatically enclose + // the text in Markdown code, so we don't need to. s += julia::ProgramCall(programName, args...); } else @@ -472,7 +473,10 @@ std::string ProgramCall(const std::string& programName, Args... args) throw std::invalid_argument("ProgramCall(): unknown " "BindingInfo::Language(): " + BindingInfo::Language() + "!"); } - s += "\n```"; + + // Close the Markdown code block, but only if we opened one. + if (BindingInfo::Language() != "julia") + s += "\n```"; return s; } diff --git a/src/mlpack/bindings/markdown/print_docs.cpp b/src/mlpack/bindings/markdown/print_docs.cpp index 8b464baa1e..2ebb67b8bd 100644 --- a/src/mlpack/bindings/markdown/print_docs.cpp +++ b/src/mlpack/bindings/markdown/print_docs.cpp @@ -126,7 +126,8 @@ void PrintDocs(const std::string& bindingName, cout << "| "; cout << ParamString(it->second.name) << " | "; cout << ParamType(it->second) << " | "; - cout << it->second.desc; // just a string + string desc = boost::replace_all_copy(it->second.desc, "|", "\\|"); + cout << desc; // just a string // Print whether or not it's a "special" language-only parameter. if (it->second.name == "copy_all_inputs" || it->second.name == "help" || it->second.name == "info" || it->second.name == "version") @@ -180,7 +181,9 @@ void PrintDocs(const std::string& bindingName, cout << "{: #" << languages[i] << "_" << bindingName << "_detailed-documentation }" << endl; cout << endl; - cout << programDoc.documentation() << endl; + string doc = boost::replace_all_copy(programDoc.documentation(), + "|", "\\|"); + cout << doc << endl; cout << endl; cout << "### See also" << endl; diff --git a/src/mlpack/core/arma_extend/Cube_extra_meat.hpp b/src/mlpack/core/arma_extend/Cube_extra_meat.hpp index a2ebd94cb4..3f33bb59d8 100644 --- a/src/mlpack/core/arma_extend/Cube_extra_meat.hpp +++ b/src/mlpack/core/arma_extend/Cube_extra_meat.hpp @@ -19,6 +19,9 @@ void Cube::serialize(Archive& ar, const unsigned int /* version */) // mem_state will always be 0 on load, so we don't need to save it. if (Archive::is_loading::value) { + // Clean any mat pointers. + delete_mat(); + // Don't free if local memory is being used. if (mem_state == 0 && mem != NULL && old_n_elem > arma_config::mat_prealloc) { diff --git a/src/mlpack/core/arma_extend/arma_extend.hpp b/src/mlpack/core/arma_extend/arma_extend.hpp index e323b993ea..ba47c1f190 100644 --- a/src/mlpack/core/arma_extend/arma_extend.hpp +++ b/src/mlpack/core/arma_extend/arma_extend.hpp @@ -36,6 +36,12 @@ #endif #endif +// Force definition of old HDF5 API. Thanks to Mike Roberts for helping find +// this workaround. +#if !defined(H5_USE_110_API) + #define H5_USE_110_API +#endif + // Include everything we'll need for serialize(). #include #include diff --git a/src/mlpack/core/boost_backport/CMakeLists.txt b/src/mlpack/core/boost_backport/CMakeLists.txt index c2b15d5120..5edfe13cab 100644 --- a/src/mlpack/core/boost_backport/CMakeLists.txt +++ b/src/mlpack/core/boost_backport/CMakeLists.txt @@ -2,18 +2,14 @@ # Anything not in this list will not be compiled into mlpack. set(SOURCES bernoulli.hpp - boost_backport_math.hpp boost_backport_serialization.hpp detail/bernoulli_details.hpp detail/polygamma.hpp detail/unchecked_bernoulli.hpp math_fwd.hpp policy.hpp - polygamma.hpp - trigamma.hpp unordered_collections_load_imp.hpp unordered_collections_save_imp.hpp - unordered_map.hpp vector.hpp string_view.hpp string_view_fwd.hpp diff --git a/src/mlpack/core/boost_backport/boost_backport_math.hpp b/src/mlpack/core/boost_backport/boost_backport_math.hpp deleted file mode 100644 index f3d7f3cc32..0000000000 --- a/src/mlpack/core/boost_backport/boost_backport_math.hpp +++ /dev/null @@ -1,29 +0,0 @@ -/** - * @file boost_backport.hpp - * @author Yannis Mentekidis - * - * Centralized control of what boost files to include. We have backported the - * following boost functionality here: - * - * * trigamma and polygamma function evaluation (added in boost 1.58.0) - * - * For versions 1.56, 1.57 we include the backported polygamma and trigamma - * functions. Anything newer, we include from Boost. - */ -#ifndef MLPACK_CORE_BOOST_BACKPORT_MATH_HPP -#define MLPACK_CORE_BOOST_BACKPORT_MATH_HPP - -#include - -#if BOOST_VERSION < 105800 - // Backported trigamma and polygamma. - #include "mlpack/core/boost_backport/trigamma.hpp" - #include "mlpack/core/boost_backport/polygamma.hpp" -#else - // Boost's version. - #include - #include -#endif - -#endif // MLPACK_CORE_BOOST_BACKPORT_HPP - diff --git a/src/mlpack/core/boost_backport/boost_backport_serialization.hpp b/src/mlpack/core/boost_backport/boost_backport_serialization.hpp index 173761f5f7..2b6c2cdd03 100644 --- a/src/mlpack/core/boost_backport/boost_backport_serialization.hpp +++ b/src/mlpack/core/boost_backport/boost_backport_serialization.hpp @@ -16,14 +16,7 @@ #define MLPACK_CORE_BOOST_BACKPORT_SERIALIZATION_HPP #include - -#if BOOST_VERSION < 105600 - // Backported unordered_map. - #include "mlpack/core/boost_backport/unordered_map.hpp" -#else - // Boost's version. - #include -#endif +#include #if BOOST_VERSION == 105800 /** diff --git a/src/mlpack/core/boost_backport/polygamma.hpp b/src/mlpack/core/boost_backport/polygamma.hpp deleted file mode 100644 index 3ff77836b3..0000000000 --- a/src/mlpack/core/boost_backport/polygamma.hpp +++ /dev/null @@ -1,94 +0,0 @@ - -/////////////////////////////////////////////////////////////////////////////// -// Copyright 2013 Nikhar Agrawal -// Copyright 2013 Christopher Kormanyos -// Copyright 2014 John Maddock -// Copyright 2013 Paul Bristow -// Distributed under the Boost -// Software License, Version 1.0. (See accompanying file -// LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#ifndef _BOOST_POLYGAMMA_2013_07_30_HPP_ - #define _BOOST_POLYGAMMA_2013_07_30_HPP_ - -#include "detail/polygamma.hpp" -#include "trigamma.hpp" -#include - -// Forward declarations -namespace boost { namespace math { - template - inline typename tools::promote_args::type - trigamma(T x, const Policy&); - - template - inline typename tools::promote_args::type - trigamma(T x); -}} - -namespace boost { namespace math { - - - template - inline typename tools::promote_args::type polygamma(const int n, T x, const Policy& pol) - { - // - // Filter off special cases right at the start: - // - if(n == 0) - return boost::math::digamma(x, pol); - if(n == 1) - return boost::math::trigamma(x, pol); - // - // We've found some standard library functions to misbehave if any FPU exception flags - // are set prior to their call, this code will clear those flags, then reset them - // on exit: - // - BOOST_FPU_EXCEPTION_GUARD - // - // The type of the result - the common type of T and U after - // any integer types have been promoted to double: - // - typedef typename tools::promote_args::type result_type; - // - // The type used for the calculation. This may be a wider type than - // the result in order to ensure full precision: - // - typedef typename policies::evaluation::type value_type; - // - // The type of the policy to forward to the actual implementation. - // We disable promotion of float and double as that's [possibly] - // happened already in the line above. Also reset to the default - // any policies we don't use (reduces code bloat if we're called - // multiple times with differing policies we don't actually use). - // Also normalise the type, again to reduce code bloat in case we're - // called multiple times with functionally identical policies that happen - // to be different types. - // - typedef typename policies::normalise< - Policy, - policies::promote_float, - policies::promote_double, - policies::discrete_quantile<>, - policies::assert_undefined<> >::type forwarding_policy; - // - // Whew. Now we can make the actual call to the implementation. - // Arguments are explicitly cast to the evaluation type, and the result - // passed through checked_narrowing_cast which handles things like overflow - // according to the policy passed: - // - return policies::checked_narrowing_cast( - detail::polygamma_imp(n, static_cast(x), forwarding_policy()), - "boost::math::polygamma<%1%>(int, %1%)"); - } - - template - inline typename tools::promote_args::type polygamma(const int n, T x) - { - return boost::math::polygamma(n, x, policies::policy<>()); - } - -} } // namespace boost::math - -#endif // _BOOST_BERNOULLI_2013_05_30_HPP_ - diff --git a/src/mlpack/core/boost_backport/trigamma.hpp b/src/mlpack/core/boost_backport/trigamma.hpp deleted file mode 100644 index ebf435289e..0000000000 --- a/src/mlpack/core/boost_backport/trigamma.hpp +++ /dev/null @@ -1,469 +0,0 @@ -// (C) Copyright John Maddock 2006. -// Use, modification and distribution are subject to the -// Boost Software License, Version 1.0. (See accompanying file -// LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#ifndef BOOST_MATH_SF_TRIGAMMA_HPP -#define BOOST_MATH_SF_TRIGAMMA_HPP - -#ifdef _MSC_VER -#pragma once -#endif - -#include "math_fwd.hpp" -#include "polygamma.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace boost{ -namespace math{ -namespace detail{ - -template -T polygamma_imp(const int n, T x, const Policy &pol); - -template -T trigamma_prec(T x, const mpl::int_<53>*, const Policy&) -{ - // Max error in interpolated form: 3.736e-017 - static const T offset = BOOST_MATH_BIG_CONSTANT(T, 53, 2.1093254089355469); - static const T P_1_2[] = { - BOOST_MATH_BIG_CONSTANT(T, 53, -1.1093280605946045), - BOOST_MATH_BIG_CONSTANT(T, 53, -3.8310674472619321), - BOOST_MATH_BIG_CONSTANT(T, 53, -3.3703848401898283), - BOOST_MATH_BIG_CONSTANT(T, 53, 0.28080574467981213), - BOOST_MATH_BIG_CONSTANT(T, 53, 1.6638069578676164), - BOOST_MATH_BIG_CONSTANT(T, 53, 0.64468386819102836), - }; - static const T Q_1_2[] = { - BOOST_MATH_BIG_CONSTANT(T, 53, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 53, 3.4535389668541151), - BOOST_MATH_BIG_CONSTANT(T, 53, 4.5208926987851437), - BOOST_MATH_BIG_CONSTANT(T, 53, 2.7012734178351534), - BOOST_MATH_BIG_CONSTANT(T, 53, 0.64468798399785611), - BOOST_MATH_BIG_CONSTANT(T, 53, -0.20314516859987728e-6), - }; - // Max error in interpolated form: 1.159e-017 - static const T P_2_4[] = { - BOOST_MATH_BIG_CONSTANT(T, 53, -0.13803835004508849e-7), - BOOST_MATH_BIG_CONSTANT(T, 53, 0.50000049158540261), - BOOST_MATH_BIG_CONSTANT(T, 53, 1.6077979838469348), - BOOST_MATH_BIG_CONSTANT(T, 53, 2.5645435828098254), - BOOST_MATH_BIG_CONSTANT(T, 53, 2.0534873203680393), - BOOST_MATH_BIG_CONSTANT(T, 53, 0.74566981111565923), - }; - static const T Q_2_4[] = { - BOOST_MATH_BIG_CONSTANT(T, 53, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 53, 2.8822787662376169), - BOOST_MATH_BIG_CONSTANT(T, 53, 4.1681660554090917), - BOOST_MATH_BIG_CONSTANT(T, 53, 2.7853527819234466), - BOOST_MATH_BIG_CONSTANT(T, 53, 0.74967671848044792), - BOOST_MATH_BIG_CONSTANT(T, 53, -0.00057069112416246805), - }; - // Maximum Deviation Found: 6.896e-018 - // Expected Error Term : -6.895e-018 - // Maximum Relative Change in Control Points : 8.497e-004 - static const T P_4_inf[] = { - static_cast(0.68947581948701249e-17L), - static_cast(0.49999999999998975L), - static_cast(1.0177274392923795L), - static_cast(2.498208511343429L), - static_cast(2.1921221359427595L), - static_cast(1.5897035272532764L), - static_cast(0.40154388356961734L), - }; - static const T Q_4_inf[] = { - static_cast(1.0L), - static_cast(1.7021215452463932L), - static_cast(4.4290431747556469L), - static_cast(2.9745631894384922L), - static_cast(2.3013614809773616L), - static_cast(0.28360399799075752L), - static_cast(0.022892987908906897L), - }; - - if(x <= 2) - { - return (offset + boost::math::tools::evaluate_polynomial(P_1_2, x) / tools::evaluate_polynomial(Q_1_2, x)) / (x * x); - } - else if(x <= 4) - { - T y = 1 / x; - return (1 + tools::evaluate_polynomial(P_2_4, y) / tools::evaluate_polynomial(Q_2_4, y)) / x; - } - T y = 1 / x; - return (1 + tools::evaluate_polynomial(P_4_inf, y) / tools::evaluate_polynomial(Q_4_inf, y)) / x; -} - -template -T trigamma_prec(T x, const mpl::int_<64>*, const Policy&) -{ - // Max error in interpolated form: 1.178e-020 - static const T offset_1_2 = BOOST_MATH_BIG_CONSTANT(T, 64, 2.109325408935546875); - static const T P_1_2[] = { - BOOST_MATH_BIG_CONSTANT(T, 64, -1.10932535608960258341), - BOOST_MATH_BIG_CONSTANT(T, 64, -4.18793841543017129052), - BOOST_MATH_BIG_CONSTANT(T, 64, -4.63865531898487734531), - BOOST_MATH_BIG_CONSTANT(T, 64, -0.919832884430500908047), - BOOST_MATH_BIG_CONSTANT(T, 64, 1.68074038333180423012), - BOOST_MATH_BIG_CONSTANT(T, 64, 1.21172611429185622377), - BOOST_MATH_BIG_CONSTANT(T, 64, 0.259635673503366427284), - }; - static const T Q_1_2[] = { - BOOST_MATH_BIG_CONSTANT(T, 64, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 64, 3.77521119359546982995), - BOOST_MATH_BIG_CONSTANT(T, 64, 5.664338024578956321), - BOOST_MATH_BIG_CONSTANT(T, 64, 4.25995134879278028361), - BOOST_MATH_BIG_CONSTANT(T, 64, 1.62956638448940402182), - BOOST_MATH_BIG_CONSTANT(T, 64, 0.259635512844691089868), - BOOST_MATH_BIG_CONSTANT(T, 64, 0.629642219810618032207e-8), - }; - // Max error in interpolated form: 3.912e-020 - static const T P_2_8[] = { - BOOST_MATH_BIG_CONSTANT(T, 64, -0.387540035162952880976e-11), - BOOST_MATH_BIG_CONSTANT(T, 64, 0.500000000276430504), - BOOST_MATH_BIG_CONSTANT(T, 64, 3.21926880986360957306), - BOOST_MATH_BIG_CONSTANT(T, 64, 10.2550347708483445775), - BOOST_MATH_BIG_CONSTANT(T, 64, 18.9002075150709144043), - BOOST_MATH_BIG_CONSTANT(T, 64, 21.0357215832399705625), - BOOST_MATH_BIG_CONSTANT(T, 64, 13.4346512182925923978), - BOOST_MATH_BIG_CONSTANT(T, 64, 3.98656291026448279118), - }; - static const T Q_2_8[] = { - BOOST_MATH_BIG_CONSTANT(T, 64, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 64, 6.10520430478613667724), - BOOST_MATH_BIG_CONSTANT(T, 64, 18.475001060603645512), - BOOST_MATH_BIG_CONSTANT(T, 64, 31.7087534567758405638), - BOOST_MATH_BIG_CONSTANT(T, 64, 31.908814523890465398), - BOOST_MATH_BIG_CONSTANT(T, 64, 17.4175479039227084798), - BOOST_MATH_BIG_CONSTANT(T, 64, 3.98749106958394941276), - BOOST_MATH_BIG_CONSTANT(T, 64, -0.000115917322224411128566), - }; - // Maximum Deviation Found: 2.635e-020 - // Expected Error Term : 2.635e-020 - // Maximum Relative Change in Control Points : 1.791e-003 - static const T P_8_inf[] = { - BOOST_MATH_BIG_CONSTANT(T, 64, -0.263527875092466899848e-19), - BOOST_MATH_BIG_CONSTANT(T, 64, 0.500000000000000058145), - BOOST_MATH_BIG_CONSTANT(T, 64, 0.0730121433777364138677), - BOOST_MATH_BIG_CONSTANT(T, 64, 1.94505878379957149534), - BOOST_MATH_BIG_CONSTANT(T, 64, 0.0517092358874932620529), - BOOST_MATH_BIG_CONSTANT(T, 64, 1.07995383547483921121), - }; - static const T Q_8_inf[] = { - BOOST_MATH_BIG_CONSTANT(T, 64, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 64, -0.187309046577818095504), - BOOST_MATH_BIG_CONSTANT(T, 64, 3.95255391645238842975), - BOOST_MATH_BIG_CONSTANT(T, 64, -1.14743283327078949087), - BOOST_MATH_BIG_CONSTANT(T, 64, 2.52989799376344914499), - BOOST_MATH_BIG_CONSTANT(T, 64, -0.627414303172402506396), - BOOST_MATH_BIG_CONSTANT(T, 64, 0.141554248216425512536), - }; - - if(x <= 2) - { - return (offset_1_2 + boost::math::tools::evaluate_polynomial(P_1_2, x) / tools::evaluate_polynomial(Q_1_2, x)) / (x * x); - } - else if(x <= 8) - { - T y = 1 / x; - return (1 + tools::evaluate_polynomial(P_2_8, y) / tools::evaluate_polynomial(Q_2_8, y)) / x; - } - T y = 1 / x; - return (1 + tools::evaluate_polynomial(P_8_inf, y) / tools::evaluate_polynomial(Q_8_inf, y)) / x; -} - -template -T trigamma_prec(T x, const mpl::int_<113>*, const Policy&) -{ - // Max error in interpolated form: 1.916e-035 - - static const T P_1_2[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, -0.999999999999999082554457936871832533), - BOOST_MATH_BIG_CONSTANT(T, 113, -4.71237311120865266379041700054847734), - BOOST_MATH_BIG_CONSTANT(T, 113, -7.94125711970499027763789342500817316), - BOOST_MATH_BIG_CONSTANT(T, 113, -5.74657746697664735258222071695644535), - BOOST_MATH_BIG_CONSTANT(T, 113, -0.404213349456398905981223965160595687), - BOOST_MATH_BIG_CONSTANT(T, 113, 2.47877781178642876561595890095758896), - BOOST_MATH_BIG_CONSTANT(T, 113, 2.07714151702455125992166949812126433), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.858877899162360138844032265418028567), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.20499222604410032375789018837922397), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.0272103140348194747360175268778415049), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.0015764849020876949848954081173520686), - }; - static const T Q_1_2[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 113, 4.71237311120863419878375031457715223), - BOOST_MATH_BIG_CONSTANT(T, 113, 9.58619118655339853449127952145877467), - BOOST_MATH_BIG_CONSTANT(T, 113, 11.0940067269829372437561421279054968), - BOOST_MATH_BIG_CONSTANT(T, 113, 8.09075424749327792073276309969037885), - BOOST_MATH_BIG_CONSTANT(T, 113, 3.87705890159891405185343806884451286), - BOOST_MATH_BIG_CONSTANT(T, 113, 1.22758678701914477836330837816976782), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.249092040606385004109672077814668716), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.0295750413900655597027079600025569048), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.00157648490200498142247694709728858139), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.161264050344059471721062360645432809e-14), - }; - - // Max error in interpolated form: 8.958e-035 - static const T P_2_4[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, -2.55843734739907925764326773972215085), - BOOST_MATH_BIG_CONSTANT(T, 113, -12.2830208240542011967952466273455887), - BOOST_MATH_BIG_CONSTANT(T, 113, -23.9195022162767993526575786066414403), - BOOST_MATH_BIG_CONSTANT(T, 113, -24.9256431504823483094158828285470862), - BOOST_MATH_BIG_CONSTANT(T, 113, -14.7979122765478779075108064826412285), - BOOST_MATH_BIG_CONSTANT(T, 113, -4.46654453928610666393276765059122272), - BOOST_MATH_BIG_CONSTANT(T, 113, -0.0191439033405649675717082465687845002), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.515412052554351265708917209749037352), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.195378348786064304378247325360320038), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.0334761282624174313035014426794245393), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.002373665205942206348500250056602687), - }; - static const T Q_2_4[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 113, 4.80098558454419907830670928248659245), - BOOST_MATH_BIG_CONSTANT(T, 113, 9.99220727843170133895059300223445265), - BOOST_MATH_BIG_CONSTANT(T, 113, 11.8896146167631330735386697123464976), - BOOST_MATH_BIG_CONSTANT(T, 113, 8.96613256683809091593793565879092581), - BOOST_MATH_BIG_CONSTANT(T, 113, 4.47254136149624110878909334574485751), - BOOST_MATH_BIG_CONSTANT(T, 113, 1.48600982028196527372434773913633152), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.319570735766764237068541501137990078), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.0407358345787680953107374215319322066), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.00237366520593271641375755486420859837), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.239554887903526152679337256236302116e-15), - BOOST_MATH_BIG_CONSTANT(T, 113, -0.294749244740618656265237072002026314e-17), - }; - - static const T y_offset_2_4 = BOOST_MATH_BIG_CONSTANT(T, 113, 3.558437347412109375); - - // Max error in interpolated form: 4.319e-035 - static const T P_4_8[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, 0.166626112697021464248967707021688845e-16), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.499999999999997739552090249208808197), - BOOST_MATH_BIG_CONSTANT(T, 113, 6.40270945019053817915772473771553187), - BOOST_MATH_BIG_CONSTANT(T, 113, 41.3833374155000608013677627389343329), - BOOST_MATH_BIG_CONSTANT(T, 113, 166.803341854562809335667241074035245), - BOOST_MATH_BIG_CONSTANT(T, 113, 453.39964786925369319960722793414521), - BOOST_MATH_BIG_CONSTANT(T, 113, 851.153712317697055375935433362983944), - BOOST_MATH_BIG_CONSTANT(T, 113, 1097.70657567285059133109286478004458), - BOOST_MATH_BIG_CONSTANT(T, 113, 938.431232478455316020076349367632922), - BOOST_MATH_BIG_CONSTANT(T, 113, 487.268001604651932322080970189930074), - BOOST_MATH_BIG_CONSTANT(T, 113, 119.953445242335730062471193124820659), - }; - static const T Q_4_8[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 113, 12.4720855670474488978638945855932398), - BOOST_MATH_BIG_CONSTANT(T, 113, 78.6093129753298570701376952709727391), - BOOST_MATH_BIG_CONSTANT(T, 113, 307.470246050318322489781182863190127), - BOOST_MATH_BIG_CONSTANT(T, 113, 805.140686101151538537565264188630079), - BOOST_MATH_BIG_CONSTANT(T, 113, 1439.12019760292146454787601409644413), - BOOST_MATH_BIG_CONSTANT(T, 113, 1735.6105285756048831268586001383127), - BOOST_MATH_BIG_CONSTANT(T, 113, 1348.32500712856328019355198611280536), - BOOST_MATH_BIG_CONSTANT(T, 113, 607.225985860570846699704222144650563), - BOOST_MATH_BIG_CONSTANT(T, 113, 119.952317857277045332558673164517227), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.000140165918355036060868680809129436084), - }; - - // Maximum Deviation Found: 2.867e-035 - // Expected Error Term : 2.866e-035 - // Maximum Relative Change in Control Points : 2.662e-004 - static const T P_8_16[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, -0.184828315274146610610872315609837439e-19), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.500000000000000004122475157735807738), - BOOST_MATH_BIG_CONSTANT(T, 113, 3.02533865247313349284875558880415875), - BOOST_MATH_BIG_CONSTANT(T, 113, 13.5995927517457371243039532492642734), - BOOST_MATH_BIG_CONSTANT(T, 113, 35.3132224283087906757037999452941588), - BOOST_MATH_BIG_CONSTANT(T, 113, 67.1639424550714159157603179911505619), - BOOST_MATH_BIG_CONSTANT(T, 113, 83.5767733658513967581959839367419891), - BOOST_MATH_BIG_CONSTANT(T, 113, 71.073491212235705900866411319363501), - BOOST_MATH_BIG_CONSTANT(T, 113, 35.8621515614725564575893663483998663), - BOOST_MATH_BIG_CONSTANT(T, 113, 8.72152231639983491987779743154333318), - }; - static const T Q_8_16[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 113, 5.71734397161293452310624822415866372), - BOOST_MATH_BIG_CONSTANT(T, 113, 25.293404179620438179337103263274815), - BOOST_MATH_BIG_CONSTANT(T, 113, 62.2619767967468199111077640625328469), - BOOST_MATH_BIG_CONSTANT(T, 113, 113.955048909238993473389714972250235), - BOOST_MATH_BIG_CONSTANT(T, 113, 130.807138328938966981862203944329408), - BOOST_MATH_BIG_CONSTANT(T, 113, 102.423146902337654110717764213057753), - BOOST_MATH_BIG_CONSTANT(T, 113, 44.0424772805245202514468199602123565), - BOOST_MATH_BIG_CONSTANT(T, 113, 8.89898032477904072082994913461386099), - BOOST_MATH_BIG_CONSTANT(T, 113, -0.0296627336872039988632793863671456398), - }; - // Maximum Deviation Found: 1.079e-035 - // Expected Error Term : -1.079e-035 - // Maximum Relative Change in Control Points : 7.884e-003 - static const T P_16_inf[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, 0.0), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.500000000000000000000000000000087317), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.345625669885456215194494735902663968), - BOOST_MATH_BIG_CONSTANT(T, 113, 9.62895499360842232127552650044647769), - BOOST_MATH_BIG_CONSTANT(T, 113, 3.5936085382439026269301003761320812), - BOOST_MATH_BIG_CONSTANT(T, 113, 49.459599118438883265036646019410669), - BOOST_MATH_BIG_CONSTANT(T, 113, 7.77519237321893917784735690560496607), - BOOST_MATH_BIG_CONSTANT(T, 113, 74.4536074488178075948642351179304121), - BOOST_MATH_BIG_CONSTANT(T, 113, 2.75209340397069050436806159297952699), - BOOST_MATH_BIG_CONSTANT(T, 113, 23.9292359711471667884504840186561598), - }; - static const T Q_16_inf[] = { - BOOST_MATH_BIG_CONSTANT(T, 113, 1.0), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.357918006437579097055656138920742037), - BOOST_MATH_BIG_CONSTANT(T, 113, 19.1386039850709849435325005484512944), - BOOST_MATH_BIG_CONSTANT(T, 113, 0.874349081464143606016221431763364517), - BOOST_MATH_BIG_CONSTANT(T, 113, 98.6516097434855572678195488061432509), - BOOST_MATH_BIG_CONSTANT(T, 113, -16.1051972833382893468655223662534306), - BOOST_MATH_BIG_CONSTANT(T, 113, 154.316860216253720989145047141653727), - BOOST_MATH_BIG_CONSTANT(T, 113, -40.2026880424378986053105969312264534), - BOOST_MATH_BIG_CONSTANT(T, 113, 60.1679136674264778074736441126810223), - BOOST_MATH_BIG_CONSTANT(T, 113, -13.3414844622256422644504472438320114), - BOOST_MATH_BIG_CONSTANT(T, 113, 2.53795636200649908779512969030363442), - }; - - if(x <= 2) - { - return (2 + boost::math::tools::evaluate_polynomial(P_1_2, x) / tools::evaluate_polynomial(Q_1_2, x)) / (x * x); - } - else if(x <= 4) - { - return (y_offset_2_4 + boost::math::tools::evaluate_polynomial(P_2_4, x) / tools::evaluate_polynomial(Q_2_4, x)) / (x * x); - } - else if(x <= 8) - { - T y = 1 / x; - return (1 + tools::evaluate_polynomial(P_4_8, y) / tools::evaluate_polynomial(Q_4_8, y)) / x; - } - else if(x <= 16) - { - T y = 1 / x; - return (1 + tools::evaluate_polynomial(P_8_16, y) / tools::evaluate_polynomial(Q_8_16, y)) / x; - } - T y = 1 / x; - return (1 + tools::evaluate_polynomial(P_16_inf, y) / tools::evaluate_polynomial(Q_16_inf, y)) / x; -} - -template -T trigamma_imp(T x, const Tag* t, const Policy& pol) -{ - // - // This handles reflection of negative arguments, and all our - // error handling, then forwards to the T-specific approximation. - // - BOOST_MATH_STD_USING // ADL of std functions. - - T result = 0; - // - // Check for negative arguments and use reflection: - // - if(x <= 0) - { - // Reflect: - T z = 1 - x; - // Argument reduction for tan: - if(floor(x) == x) - { - return policies::raise_pole_error("boost::math::trigamma<%1%>(%1%)", 0, (1-x), pol); - } - T s = fabs(x) < fabs(z) ? boost::math::sin_pi(x, pol) : boost::math::sin_pi(z, pol); - return -trigamma_imp(z, t, pol) + boost::math::pow<2>(constants::pi()) / (s * s); - } - if(x < 1) - { - result = 1 / (x * x); - x += 1; - } - return result + trigamma_prec(x, t, pol); -} - -template -T trigamma_imp(T x, const mpl::int_<0>*, const Policy& pol) -{ - return polygamma_imp(1, x, pol); -} -// -// Initializer: ensure all our constants are initialized prior to the first call of main: -// -template -struct trigamma_initializer -{ - struct init - { - init() - { - typedef typename policies::precision::type precision_type; - do_init(mpl::bool_()); - } - void do_init(const mpl::true_&) - { - boost::math::trigamma(T(2.5), Policy()); - } - void do_init(const mpl::false_&){} - void force_instantiate()const{} - }; - static const init initializer; - static void force_instantiate() - { - initializer.force_instantiate(); - } -}; - -template -const typename trigamma_initializer::init trigamma_initializer::initializer; - -} // namespace detail - -template -inline typename tools::promote_args::type - trigamma(T x, const Policy&) -{ - typedef typename tools::promote_args::type result_type; - typedef typename policies::evaluation::type value_type; - typedef typename policies::precision::type precision_type; - typedef typename mpl::if_< - mpl::or_< - mpl::less_equal >, - mpl::greater > - >, - mpl::int_<0>, - typename mpl::if_< - mpl::less >, - mpl::int_<53>, - typename mpl::if_< - mpl::less >, - mpl::int_<64>, - mpl::int_<113> - >::type - >::type - >::type tag_type; - - typedef typename policies::normalise< - Policy, - policies::promote_float, - policies::promote_double, - policies::discrete_quantile<>, - policies::assert_undefined<> >::type forwarding_policy; - - // Force initialization of constants: - detail::trigamma_initializer::force_instantiate(); - - return policies::checked_narrowing_cast(detail::trigamma_imp( - static_cast(x), - static_cast(0), forwarding_policy()), "boost::math::trigamma<%1%>(%1%)"); -} - -template -inline typename tools::promote_args::type - trigamma(T x) -{ - return trigamma(x, policies::policy<>()); -} - -} // namespace math -} // namespace boost -#endif - diff --git a/src/mlpack/core/boost_backport/unordered_map.hpp b/src/mlpack/core/boost_backport/unordered_map.hpp deleted file mode 100644 index 4ae37d22c9..0000000000 --- a/src/mlpack/core/boost_backport/unordered_map.hpp +++ /dev/null @@ -1,234 +0,0 @@ -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) -#ifndef BOOST_SERIALIZATION_UNORDERED_MAP_HPP -#define BOOST_SERIALIZATION_UNORDERED_MAP_HPP - -// MS compatible compilers support #pragma once -#if defined(_MSC_VER) && (_MSC_VER >= 1020) -# pragma once -#endif - -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// serialization/unordered_map.hpp: -// serialization for stl unordered_map templates - -// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . -// (C) Copyright 2014 Jim Bell -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. - -#include - -#include - -#include -#include "unordered_collections_save_imp.hpp" -#include "unordered_collections_load_imp.hpp" -#include - -namespace boost { -namespace serialization { - -namespace stl { - -// map input -template -struct archive_input_unordered_map -{ - inline void operator()( - Archive &ar, - Container &s, - const unsigned int v - ){ - typedef typename Container::value_type type; - detail::stack_construct t(ar, v); - // borland fails silently w/o full namespace - ar >> boost::serialization::make_nvp("item", t.reference()); - std::pair result = - s.insert(t.reference()); - // note: the following presumes that the map::value_type was NOT tracked - // in the archive. This is the usual case, but here there is no way - // to determine that. - if(result.second){ - ar.reset_object_address( - & (result.first->second), - & t.reference().second - ); - } - } -}; - -// multimap input -template -struct archive_input_unordered_multimap -{ - inline void operator()( - Archive &ar, - Container &s, - const unsigned int v - ){ - typedef typename Container::value_type type; - detail::stack_construct t(ar, v); - // borland fails silently w/o full namespace - ar >> boost::serialization::make_nvp("item", t.reference()); - typename Container::const_iterator result - = s.insert(t.reference()); - // note: the following presumes that the map::value_type was NOT tracked - // in the archive. This is the usual case, but here there is no way - // to determine that. - ar.reset_object_address( - & result->second, - & t.reference() - ); - } -}; - -} // stl - -template< - class Archive, - class Key, - class HashFcn, - class EqualKey, - class Allocator -> -inline void save( - Archive & ar, - const std::unordered_map< - Key, HashFcn, EqualKey, Allocator - > &t, - const unsigned int /*file_version*/ -){ - boost::serialization::stl::save_unordered_collection< - Archive, - std::unordered_map< - Key, HashFcn, EqualKey, Allocator - > - >(ar, t); -} - -template< - class Archive, - class Key, - class HashFcn, - class EqualKey, - class Allocator -> -inline void load( - Archive & ar, - std::unordered_map< - Key, HashFcn, EqualKey, Allocator - > &t, - const unsigned int /*file_version*/ -){ - boost::serialization::stl::load_unordered_collection< - Archive, - std::unordered_map< - Key, HashFcn, EqualKey, Allocator - >, - boost::serialization::stl::archive_input_unordered_map< - Archive, - std::unordered_map< - Key, HashFcn, EqualKey, Allocator - > - > - >(ar, t); -} - -// split non-intrusive serialization function member into separate -// non intrusive save/load member functions -template< - class Archive, - class Key, - class HashFcn, - class EqualKey, - class Allocator -> -inline void serialize( - Archive & ar, - std::unordered_map< - Key, HashFcn, EqualKey, Allocator - > &t, - const unsigned int file_version -){ - boost::serialization::split_free(ar, t, file_version); -} - -// unordered_multimap -template< - class Archive, - class Key, - class HashFcn, - class EqualKey, - class Allocator -> -inline void save( - Archive & ar, - const std::unordered_multimap< - Key, HashFcn, EqualKey, Allocator - > &t, - const unsigned int /*file_version*/ -){ - boost::serialization::stl::save_unordered_collection< - Archive, - std::unordered_multimap< - Key, HashFcn, EqualKey, Allocator - > - >(ar, t); -} - -template< - class Archive, - class Key, - class HashFcn, - class EqualKey, - class Allocator -> -inline void load( - Archive & ar, - std::unordered_multimap< - Key, HashFcn, EqualKey, Allocator - > &t, - const unsigned int /*file_version*/ -){ - boost::serialization::stl::load_unordered_collection< - Archive, - std::unordered_multimap< - Key, HashFcn, EqualKey, Allocator - >, - boost::serialization::stl::archive_input_unordered_multimap< - Archive, - std::unordered_multimap< - Key, HashFcn, EqualKey, Allocator - > - > - >(ar, t); -} - -// split non-intrusive serialization function member into separate -// non intrusive save/load member functions -template< - class Archive, - class Key, - class HashFcn, - class EqualKey, - class Allocator -> -inline void serialize( - Archive & ar, - std::unordered_multimap< - Key, HashFcn, EqualKey, Allocator - > &t, - const unsigned int file_version -){ - boost::serialization::split_free(ar, t, file_version); -} - -} // namespace serialization -} // namespace boost - -#endif // BOOST_SERIALIZATION_UNORDERED_MAP_HPP diff --git a/src/mlpack/core/cv/metrics/CMakeLists.txt b/src/mlpack/core/cv/metrics/CMakeLists.txt index 4cf1027874..b9edacaf9a 100644 --- a/src/mlpack/core/cv/metrics/CMakeLists.txt +++ b/src/mlpack/core/cv/metrics/CMakeLists.txt @@ -13,6 +13,8 @@ set(SOURCES precision_impl.hpp recall.hpp recall_impl.hpp + r2_score.hpp + r2_score_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/core/cv/metrics/r2_score.hpp b/src/mlpack/core/cv/metrics/r2_score.hpp new file mode 100644 index 0000000000..6fcac955aa --- /dev/null +++ b/src/mlpack/core/cv/metrics/r2_score.hpp @@ -0,0 +1,76 @@ +/** + * @file r2_score.hpp + * @author Bisakh Mondal + * + * The R^2 (Coefficient of determination) regression metric. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_R2SCORE_HPP +#define MLPACK_CORE_CV_METRICS_R2SCORE_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * The R2 Score is a metric of performance for regression algorithms + * that represents the proportion of variance (here y) that has been + * explained by the independent variables in the model. It provides + * an indication of goodness of fit and therefore a measure of how + * well unseen samples are likely to be predicted by the model, + * through the proportion of explained variance. + * As R2 Score is dataset dependent it can have wide range of values. The + * best possible score is @f$R^2 =1.0@f$. Values of R2 outside the range + * 0 to 1 can occur when the model fits the data worse than a horizontal + * hyperplane. This would occur when the wrong model was chosen, or + * nonsensical constraints were applied by mistake. A model which + * predicts exactly the expected value of y, disregarding the input + * features, gets a R2 Score equals to 0.0. + * If a model predicts @f$ \hat{y}_i $@f of the @f$ i $@f-th sample for a true + * @f$ y_i $@f for total n samples, the R2 Score is calculated by + * @f{eqnarray*}{ + * R^{2} \left( y, \hat{y} \right) &=& 1-\frac{\sum_{i=1}^{n} + * \left( y_i - \hat{y_i} \right)^2 } + * {\sum_{i=1}^{n} \left( y_i - \bar{y}\right)^2}\\ + * @f} + * + * where @f$ \bar{y} = frac{1}{y}\sum_{i=1}^{n} y_i $@f. + * For example, a model having R2Score = 0.85, explains 85 \% variability of + * the response data around its mean. + */ +class R2Score +{ + public: + /** + * Run prediction and calculate the R squared error. + * + * @param model A regression model. + * @param data Column-major data containing test items. + * @param responses Ground truth (correct) target values for the test items, + * should be either a row vector or a column-major matrix. + * @return calculated R2 Score. + */ + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to maximize the measurement. + */ + static const bool NeedsMinimization = false; +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "r2_score_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/r2_score_impl.hpp b/src/mlpack/core/cv/metrics/r2_score_impl.hpp new file mode 100644 index 0000000000..86c57e11fb --- /dev/null +++ b/src/mlpack/core/cv/metrics/r2_score_impl.hpp @@ -0,0 +1,55 @@ +/** + * @file r2_score_impl.hpp + * @author Bisakh Mondal + * + * The implementation of the class R2Score. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_R2SCORE_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_R2SCORE_IMPL_HPP + +namespace mlpack { +namespace cv { + +template +double R2Score::Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses) +{ + if (data.n_cols != responses.n_cols) + { + std::ostringstream oss; + oss << "R2Score::Evaluate(): number of points (" << data.n_cols << ") " + << "does not match number of responses (" << responses.n_cols << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + ResponsesType predictedResponses; + // Taking Predicted Output from the model. + model.Predict(data, predictedResponses); + // Mean value of response. + double meanResponses = arma::mean(responses); + + // Calculate the numerator i.e. residual sum of squares. + double residualSumSquared = arma::accu(arma::square(responses - + predictedResponses)); + + // Calculate the denominator i.e.total sum of squares. + double totalSumSquared = arma::accu(arma::square(responses - meanResponses)); + + // Handling undefined R2 Score when both denominator and numerator is 0.0. + if (residualSumSquared == 0.0) + return totalSumSquared ? 1.0 : DBL_MIN; + + return 1 - residualSumSquared / totalSumSquared; +} + +} // namespace cv +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/data/CMakeLists.txt b/src/mlpack/core/data/CMakeLists.txt index d21c27da75..0ef68d4a94 100644 --- a/src/mlpack/core/data/CMakeLists.txt +++ b/src/mlpack/core/data/CMakeLists.txt @@ -11,6 +11,7 @@ set(SOURCES load_csv.cpp load.hpp load_image_impl.hpp + load_image.cpp load_model_impl.hpp load_vec_impl.hpp load_impl.hpp @@ -21,6 +22,7 @@ set(SOURCES normalize_labels_impl.hpp save.hpp save_impl.hpp + save_image.cpp serialization_template_version.hpp split_data.hpp imputer.hpp diff --git a/src/mlpack/core/data/image_info.hpp b/src/mlpack/core/data/image_info.hpp index e7f444447c..3fa91ef303 100644 --- a/src/mlpack/core/data/image_info.hpp +++ b/src/mlpack/core/data/image_info.hpp @@ -13,28 +13,12 @@ #ifndef MLPACK_CORE_DATA_IMAGE_INFO_HPP #define MLPACK_CORE_DATA_IMAGE_INFO_HPP - #include - #include "extension.hpp" -#ifdef HAS_STB // Compile this only if stb is present. - -#define STB_IMAGE_STATIC -#define STB_IMAGE_IMPLEMENTATION -#include - -#define STB_IMAGE_WRITE_STATIC -#define STB_IMAGE_WRITE_IMPLEMENTATION -#include - -#endif - namespace mlpack { namespace data { -#ifdef HAS_STB // Compile this only if stb is present. - /** * Checks if the given image filename is supported. * @@ -44,8 +28,6 @@ namespace data { inline bool ImageFormatSupported(const std::string& fileName, const bool save = false); -#endif - /** * Implements meta-data of images required by data::Load and * data::Save for loading and saving images into arma::Mat. @@ -87,6 +69,15 @@ class ImageInfo //! Modify the image quality. size_t& Quality() { return quality; } + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(width); + ar & BOOST_SERIALIZATION_NVP(channels); + ar & BOOST_SERIALIZATION_NVP(height); + ar & BOOST_SERIALIZATION_NVP(quality); + } + private: // To store the image width. size_t width; diff --git a/src/mlpack/core/data/load.hpp b/src/mlpack/core/data/load.hpp index 7974227409..256163f72b 100644 --- a/src/mlpack/core/data/load.hpp +++ b/src/mlpack/core/data/load.hpp @@ -291,7 +291,6 @@ bool Load(const std::string& filename, /** * Image load/save interfaces. */ -#ifdef HAS_STB /** * Load the image file into the given matrix. @@ -300,15 +299,13 @@ bool Load(const std::string& filename, * @param matrix Matrix to load the image into. * @param info An object of ImageInfo class. * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, transpose the matrix after loading. * @return Boolean value indicating success or failure of load. */ template bool Load(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); /** * Load the image file into the given matrix. @@ -324,10 +321,13 @@ template bool Load(const std::vector& files, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); -#endif // HAS_STB. +// Implementation found in load_image.cpp. +bool LoadImage(const std::string& filename, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal = false); } // namespace data } // namespace mlpack diff --git a/src/mlpack/core/data/load_image.cpp b/src/mlpack/core/data/load_image.cpp new file mode 100644 index 0000000000..2796e9a1f4 --- /dev/null +++ b/src/mlpack/core/data/load_image.cpp @@ -0,0 +1,125 @@ +/** + * @file load_image.cpp + * @author Mehul Kumar Nirala + * + * Implementation of image loading functionality via STB. + */ +#include "load.hpp" +#include "image_info.hpp" + +#ifdef HAS_STB + +#define STB_IMAGE_STATIC +#define STB_IMAGE_IMPLEMENTATION +#include + +#define STB_IMAGE_WRITE_STATIC +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include + +namespace mlpack { +namespace data { + +bool LoadImage(const std::string& filename, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal) +{ + unsigned char* image; + + if (!ImageFormatSupported(filename)) + { + std::ostringstream oss; + oss << "Load(): file type " << Extension(filename) << " not supported. "; + oss << "Currently it supports: "; + for (auto extension : loadFileTypes) + oss << " " << extension; + oss << "." << std::endl; + + if (fatal) + { + Log::Fatal << oss.str(); + } + else + { + Log::Warn << oss.str(); + } + + return false; + } + + // Temporary variables needed as stb_image.h supports int parameters. + int tempWidth, tempHeight, tempChannels; + + // For grayscale images. + if (info.Channels() == 1) + { + image = stbi_load(filename.c_str(), &tempWidth, &tempHeight, &tempChannels, + STBI_grey); + } + else + { + image = stbi_load(filename.c_str(), &tempWidth, &tempHeight, &tempChannels, + STBI_rgb); + } + + if (!image) + { + if (fatal) + { + Log::Fatal << "Load(): failed to load image '" << filename << "': " + << stbi_failure_reason() << std::endl; + } + else + { + Log::Warn << "Load(): failed to load image '" << filename << "': " + << stbi_failure_reason() << std::endl; + } + + return false; + } + + info.Width() = tempWidth; + info.Height() = tempHeight; + info.Channels() = tempChannels; + + // Copy image into armadillo Mat. + matrix = arma::Mat(image, info.Width() * info.Height() * + info.Channels(), 1, true, true); + + // Free the image pointer. + free(image); + return true; +} + +} // namespace data +} // namespace mlpack + +#else + +namespace mlpack { +namespace data { + +bool LoadImage(const std::string& /* filename */, + arma::Mat& /* matrix */, + ImageInfo& /* info */, + const bool fatal) +{ + if (fatal) + { + Log::Fatal << "Load(): mlpack was not compiled with STB support, so images " + << "cannot be loaded!" << std::endl; + } + else + { + Log::Warn << "Load(): mlpack was not compiled with STB support, so images " + << "cannot be loaded!" << std::endl; + } + + return false; +} + +} // namespace data +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/data/load_image_impl.hpp b/src/mlpack/core/data/load_image_impl.hpp index 23e45be40c..b5be00861b 100644 --- a/src/mlpack/core/data/load_image_impl.hpp +++ b/src/mlpack/core/data/load_image_impl.hpp @@ -19,68 +19,28 @@ namespace mlpack { namespace data { -#ifdef HAS_STB // Compile this only if stb is present. - // Image loading API. template bool Load(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool /* fatal */, - const bool transpose) + const bool fatal) { Timer::Start("loading_image"); - unsigned char* image; - if (!ImageFormatSupported(filename)) + // STB loads into unsigned char matrices, so we may have to convert once + // loaded. + arma::Mat tempMatrix; + const bool result = LoadImage(filename, tempMatrix, info, fatal); + + // If fatal is true, then the program will have already thrown an exception. + if (!result) { - std::ostringstream oss; - oss << "File type " << Extension(filename) << " not supported.\n"; - oss << "Currently it supports "; - for (auto extension : loadFileTypes) - oss << " " << extension; - oss << std::endl; - throw std::runtime_error(oss.str()); + Timer::Stop("loading_image"); return false; } - stbi_set_flip_vertically_on_load(transpose); - - // Temporary variables needed as stb_image.h supports int parameters. - int tempWidth, tempHeight, tempChannels; - - // For grayscale images. - if (info.Channels() == 1) - { - image = stbi_load(filename.c_str(), &tempWidth, &tempHeight, &tempChannels, - STBI_grey); - } - else - { - image = stbi_load(filename.c_str(), &tempWidth, &tempHeight, &tempChannels, - STBI_rgb); - } - - if (tempWidth <= 0 || tempHeight <= 0) - { - std::ostringstream oss; - oss << "Image '" << filename << "' not found." << std::endl; - free(image); - throw std::runtime_error(oss.str()); - - return false; - } - - info.Width() = tempWidth; - info.Height() = tempHeight; - info.Channels() = tempChannels; - - // Copy image into armadillo Mat. - matrix = arma::Mat(image, info.Width() * info.Height() * - info.Channels(), 1, true, true); - - // Free the image pointer. - free(image); + matrix = arma::conv_to>::from(tempMatrix); Timer::Stop("loading_image"); return true; } @@ -90,58 +50,46 @@ template bool Load(const std::vector& files, arma::Mat& matrix, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { if (files.size() == 0) { std::ostringstream oss; - oss << "Files vector is empty." << std::endl; + oss << "Load(): vector of image files is empty." << std::endl; + + if (fatal) + Log::Fatal << oss.str(); + else + Log::Warn << oss.str(); - throw std::runtime_error(oss.str()); return false; } arma::Mat img; - bool status = Load(files[0], img, info, fatal, transpose); + bool status = LoadImage(files[0], img, info, fatal); + + if (!status) + return false; // Decide matrix dimension using the image height and width. - matrix.set_size(info.Width() * info.Height() * info.Channels(), files.size()); - matrix.col(0) = img; + arma::Mat tmpMatrix( + info.Width() * info.Height() * info.Channels(), files.size()); + tmpMatrix.col(0) = img; for (size_t i = 1; i < files.size() ; i++) { - arma::Mat colImg(matrix.colptr(i), matrix.n_rows, 1, + arma::Mat colImg(tmpMatrix.colptr(i), tmpMatrix.n_rows, 1, false, true); - status &= Load(files[i], colImg, info, fatal, transpose); + status = LoadImage(files[i], colImg, info, fatal); + + if (!status) + return false; } - return status; -} -#else // No STB. -template -bool Load(const std::string& filename, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal = false, - const bool transpose = true) -{ - throw std::runtime_error("Load(): HAS_STB is not defined, " - "so STB is not available and images cannot be loaded!"); + matrix = arma::conv_to>::from(tmpMatrix); + return true; } -template -bool Load(const std::vector& files, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal = false, - const bool transpose = true) -{ - throw std::runtime_error("Load(): HAS_STB is not defined, " - "so STB is not available and images cannot be loaded!"); -} -#endif // HAS_STB. - } // namespace data } // namespace mlpack diff --git a/src/mlpack/core/data/save.hpp b/src/mlpack/core/data/save.hpp index 121265ac81..0fb3fe8438 100644 --- a/src/mlpack/core/data/save.hpp +++ b/src/mlpack/core/data/save.hpp @@ -91,8 +91,6 @@ bool Save(const std::string& filename, const bool fatal = false, format f = format::autodetect); -#ifdef HAS_STB - /** * Save the image file from the given matrix. * @@ -107,8 +105,7 @@ template bool Save(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); /** * Save the image file from the given matrix. @@ -124,10 +121,15 @@ template bool Save(const std::vector& files, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); -#endif // HAS_STB. +/** + * Helper function to save files. Implementation in save_image.cpp. + */ +bool SaveImage(const std::string& filename, + arma::Mat& image, + ImageInfo& info, + const bool fatal = false); } // namespace data } // namespace mlpack diff --git a/src/mlpack/core/data/save_image.cpp b/src/mlpack/core/data/save_image.cpp new file mode 100644 index 0000000000..2c4654b711 --- /dev/null +++ b/src/mlpack/core/data/save_image.cpp @@ -0,0 +1,141 @@ +/** + * @file save_image.cpp + * @author Mehul Kumar Nirala + * + * Implementation of image saving functionality via STB. + */ +#include "save.hpp" + +#ifdef HAS_STB + +#define STB_IMAGE_STATIC +#define STB_IMAGE_IMPLEMENTATION +#include + +#define STB_IMAGE_WRITE_STATIC +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include + +namespace mlpack { +namespace data { + +bool SaveImage(const std::string& filename, + arma::Mat& image, + ImageInfo& info, + const bool fatal) +{ + // Check to see if the file type is supported. + if (!ImageFormatSupported(filename, true)) + { + std::ostringstream oss; + oss << "Save(): file type " << Extension(filename) << " not supported.\n"; + oss << "Currently image saving supports "; + for (auto extension : saveFileTypes) + oss << ", " << extension; + oss << "." << std::endl; + + if (fatal) + { + Log::Fatal << oss.str(); + } + else + { + Log::Warn << oss.str(); + } + + return false; + } + + // Ensure the shape of the matrix is correct. + if (image.n_cols > 1) + { + Log::Warn << "Save(): given input image matrix contains more than 1 image." + << std::endl; + Log::Warn << "Only the first image will be saved!" << std::endl; + } + + if (info.Width() * info.Height() * info.Channels() != image.n_elem) + { + Log::Fatal << "data::Save(): The given image dimensions do not match the " + << "dimensions of the matrix to be saved!" << std::endl; + } + + bool status = false; + unsigned char* imageMem = image.memptr(); + + if ("png" == Extension(filename)) + { + status = stbi_write_png(filename.c_str(), info.Width(), info.Height(), + info.Channels(), imageMem, info.Width() * info.Channels()); + } + else if ("bmp" == Extension(filename)) + { + status = stbi_write_bmp(filename.c_str(), info.Width(), info.Height(), + info.Channels(), imageMem); + } + else if ("tga" == Extension(filename)) + { + status = stbi_write_tga(filename.c_str(), info.Width(), info.Height(), + info.Channels(), imageMem); + } + else if ("hdr" == Extension(filename)) + { + // We'll have to convert to float... + arma::fmat tmpImage = arma::conv_to::from(image); + status = stbi_write_hdr(filename.c_str(), info.Width(), info.Height(), + info.Channels(), tmpImage.memptr()); + } + else if ("jpg" == Extension(filename)) + { + status = stbi_write_jpg(filename.c_str(), info.Width(), info.Height(), + info.Channels(), imageMem, info.Quality()); + } + + if (!status) + { + if (fatal) + { + Log::Fatal << "Save(): error saving image to '" << filename << "'." + << std::endl; + } + else + { + Log::Warn << "Save(): error saving image to '" << filename << "'." + << std::endl; + } + } + + return status; +} + +} // namespace data +} // namespace mlpack + +#else + +namespace mlpack { +namespace data { + +bool SaveImage(const std::string& /* filename */, + arma::Mat& /* image */, + ImageInfo& /* info */, + const bool fatal) +{ + if (fatal) + { + Log::Fatal << "Save(): mlpack was not compiled with STB support, so images " + << "cannot be saved!" << std::endl; + } + else + { + Log::Warn << "Save(): mlpack was not compiled with STB support, so images " + << "cannot be saved!" << std::endl; + } + + return false; +} + +} // namespace data +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index 18a61530a5..516e0aaf93 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -283,86 +283,25 @@ bool Save(const std::string& filename, } } -#ifdef HAS_STB -// Image saving API. +/** + * Save the given image to the given filename. + * + * @param filename Filename to save to. + * @param matrix Matrix containing image to be saved. + * @param info Information about the image (width/height/channels/etc.). + * @param fatal Whether an exception should be thrown on save failure. + */ template bool Save(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { - Timer::Start("saving_image"); - // We transpose by default. So, un-transpose if necessary. - if (!transpose) - matrix = arma::trans(matrix); + arma::Mat tmpMatrix = + arma::conv_to>::from(matrix); - int tempWidth, tempHeight, tempChannels, tempQuality; - - tempWidth = info.Width(); - tempHeight = info.Height(); - tempChannels = info.Channels(); - tempQuality = info.Quality(); - - if (!ImageFormatSupported(filename, true)) - { - std::ostringstream oss; - oss << "File type " << Extension(filename) << " not supported.\n"; - oss << "Currently it supports "; - for (auto extension : saveFileTypes) - oss << ", " << extension; - oss << std::endl; - throw std::runtime_error(oss.str()); - return false; - } - if (matrix.n_cols > 1) - { - std::cout << "Input Matrix contains more than 1 image." << std::endl; - std::cout << "Only the firstimage will be saved!" << std::endl; - } - stbi_flip_vertically_on_write(transpose); - - bool status = false; - try - { - unsigned char* image = matrix.memptr(); - - if ("png" == Extension(filename)) - { - status = stbi_write_png(filename.c_str(), tempWidth, tempHeight, - tempChannels, image, tempWidth * tempChannels); - } - else if ("bmp" == Extension(filename)) - { - status = stbi_write_bmp(filename.c_str(), tempWidth, tempHeight, - tempChannels, image); - } - else if ("tga" == Extension(filename)) - { - status = stbi_write_tga(filename.c_str(), tempWidth, tempHeight, - tempChannels, image); - } - else if ("hdr" == Extension(filename)) - { - status = stbi_write_hdr(filename.c_str(), tempWidth, tempHeight, - tempChannels, reinterpret_cast(image)); - } - else if ("jpg" == Extension(filename)) - { - status = stbi_write_jpg(filename.c_str(), tempWidth, tempHeight, - tempChannels, image, tempQuality); - } - } - catch (std::exception& e) - { - Timer::Stop("saving_image"); - if (fatal) - Log::Fatal << e.what() << std::endl; - Log::Warn << e.what() << std::endl; - return false; - } - Timer::Stop("saving_image"); - return status; + // Call out to .cpp implementation. + return SaveImage(filename, tmpMatrix, info, fatal); } // Image saving API for multiple files. @@ -370,59 +309,36 @@ template bool Save(const std::vector& files, arma::Mat& matrix, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { if (files.size() == 0) { - std::ostringstream oss; - oss << "Files vector is empty." << std::endl; + if (fatal) + { + Log::Fatal << "Save(): vector of image files is empty; nothing to save." + << std::endl; + } + else + { + Log::Warn << "Save(): vector of image files is empty; nothing to save." + << std::endl; + } - throw std::runtime_error(oss.str()); return false; } - // We transpose by default. So, un-transpose if necessary. - if (!transpose) - matrix = arma::trans(matrix); arma::Mat img; - bool status = Save(files[0], img, info, fatal, transpose); + bool status = true; - // Decide matrix dimension using the image height and width. - matrix.set_size(info.Width() * info.Height() * info.Channels(), files.size()); - matrix.col(0) = img; - - for (size_t i = 1; i < files.size() ; i++) + for (size_t i = 0; i < files.size() ; i++) { - arma::Mat colImg(matrix.colptr(i), matrix.n_rows, 1, + arma::Mat colImg(matrix.colptr(i), matrix.n_rows, 1, false, true); - status &= Save(files[i], colImg, info, fatal, transpose); + status &= Save(files[i], colImg, info, fatal); } + return status; } -#else -template -bool Save(const std::string& filename, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal = false, - const bool transpose = true) -{ - throw std::runtime_error("Save(): HAS_STB is not defined, " - "so STB is not available and images cannot be saved!"); -} - -template -bool Save(const std::vector& files, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal = false, - const bool transpose = true) -{ - throw std::runtime_error("Save(): HAS_STB is not defined, " - "so STB is not available and images cannot be saved!"); -} -#endif // HAS_STB. } // namespace data } // namespace mlpack diff --git a/src/mlpack/core/data/scaler_methods/zca_whitening.hpp b/src/mlpack/core/data/scaler_methods/zca_whitening.hpp index 0c78fb3495..80638de954 100644 --- a/src/mlpack/core/data/scaler_methods/zca_whitening.hpp +++ b/src/mlpack/core/data/scaler_methods/zca_whitening.hpp @@ -52,10 +52,7 @@ class ZCAWhitening * * @param eps Regularization parameter. */ - ZCAWhitening(double eps = 0.00005) - { - pca = new data::PCAWhitening(eps); - } + ZCAWhitening(double eps = 0.00005) : pca(eps) { } /** * Function to fit features, to find out the min max and scale. @@ -65,7 +62,7 @@ class ZCAWhitening template void Fit(const MatType& input) { - pca->Fit(input); + pca.Fit(input); } /** @@ -77,8 +74,8 @@ class ZCAWhitening template void Transform(const MatType& input, MatType& output) { - pca->Transform(input, output); - output = pca->EigenVectors() * output; + pca.Transform(input, output); + output = pca.EigenVectors() * output; } /** @@ -90,19 +87,19 @@ class ZCAWhitening template void InverseTransform(const MatType& input, MatType& output) { - output = inv(pca->EigenVectors()) * arma::diagmat(arma::sqrt( - pca->EigenValues())) * inv(pca->EigenVectors().t()) * input; - output = (output.each_col() + pca->ItemMean()); + output = inv(pca.EigenVectors()) * arma::diagmat(arma::sqrt( + pca.EigenValues())) * inv(pca.EigenVectors().t()) * input; + output = (output.each_col() + pca.ItemMean()); } //! Get the mean row vector. - const arma::vec& ItemMean() const { return pca->ItemMean(); } + const arma::vec& ItemMean() const { return pca.ItemMean(); } //! Get the eigenvalues vector. - const arma::vec& EigenValues() const { return pca->EigenValues(); } + const arma::vec& EigenValues() const { return pca.EigenValues(); } //! Get the eigenvector. - const arma::mat& EigenVectors() const { return pca->EigenVectors(); } + const arma::mat& EigenVectors() const { return pca.EigenVectors(); } //! Get the regularization parameter. - double Epsilon() const { return pca->Epsilon(); } + double Epsilon() const { return pca.Epsilon(); } template void serialize(Archive& ar, const unsigned int /* version */) @@ -112,7 +109,7 @@ class ZCAWhitening private: // A pointer to PcaWhitening Class. - PCAWhitening* pca; + PCAWhitening pca; }; // class ZCAWhitening } // namespace data diff --git a/src/mlpack/core/data/string_encoding.hpp b/src/mlpack/core/data/string_encoding.hpp index 6c529fe31f..9ed18cba6e 100644 --- a/src/mlpack/core/data/string_encoding.hpp +++ b/src/mlpack/core/data/string_encoding.hpp @@ -24,7 +24,8 @@ namespace data { /** * The class translates a set of strings into numbers using various encoding - * algorithms. + * algorithms. The encoder writes data either in the column-major order or + * in the row-major order depending on the output data type. * * @tparam EncodingPolicyType Type of the encoding algorithm itself. * @tparam DictionaryType Type of the dictionary. @@ -90,11 +91,17 @@ class StringEncoding void Clear(); /** - * Encode the given text and write the result to the given output. + * Encode the given text and write the result to the given output. The encoder + * writes data in the column-major order or in the row-major order depending + * on the output data type. + * + * If the output type is either arma::mat or arma::sp_mat then the function + * writes it in the column-major order. If the output type is 2D std::vector + * then the function writes it in the row major order. * * @tparam OutputType Type of the output container. The function supports * the following types: arma::mat, arma::sp_mat, - * std::vector>. + * std::vector>. * @tparam TokenizerType Type of the tokenizer. * * @param input Corpus of text to encode. @@ -132,11 +139,16 @@ class StringEncoding private: /** * A helper function to encode the given text and write the result to - * the given output. + * the given output. The encoder writes data in the column-major order or + * in the row-major order depending on the output data type. + * + * If the output type is either arma::mat or arma::sp_mat then the function + * writes it in the column-major order. If the output type is 2D std::vector + * then the function writes it in the row major order. * * @tparam OutputType Type of the output container. The function supports * the following types: arma::mat, arma::sp_mat, - * std::vector>. + * std::vector>. * @tparam TokenizerType Type of the tokenizer. * @tparam PolicyType The type of the encoding policy. It has to be * equal to EncodingPolicyType. @@ -153,9 +165,7 @@ class StringEncoding * 2. IsTokenEmpty() that accepts a token and returns true if the given * token is empty. */ - template + template void EncodeHelper(const std::vector& input, OutputType& output, const TokenizerType& tokenizer, @@ -164,11 +174,13 @@ class StringEncoding /** * A helper function to encode the given text and write the result to * the given output. This is an optimized overload for policies that support - * the one pass encoding algorithm. + * the one pass encoding algorithm. The encoder writes data in the row-major + * order. * * @tparam TokenizerType Type of the tokenizer. * @tparam PolicyType The type of the encoding policy. It has to be * equal to EncodingPolicyType. + * @tparam ElemType Type of the output values. * * @param input Corpus of text to encode. * @param output Output container to store the result. @@ -182,9 +194,9 @@ class StringEncoding * 2. IsTokenEmpty() that accepts a token and returns true if the given * token is empty. */ - template + template void EncodeHelper(const std::vector& input, - std::vector>& output, + std::vector>& output, const TokenizerType& tokenizer, PolicyType& policy, typename std::enable_if& input, { size_t numColumns = 0; + policy.Reset(); + // The first pass adds the extracted tokens to the dictionary. - for (const std::string& line : input) + for (size_t i = 0; i < input.size(); i++) { - boost::string_view strView(line); + boost::string_view strView(input[i]); auto token = tokenizer(strView); static_assert( @@ -127,9 +129,12 @@ EncodeHelper(const std::vector& input, if (!dictionary.HasToken(token)) dictionary.AddToken(std::move(token)); + policy.PreprocessToken(i, numTokens, dictionary.Value(token)); + token = tokenizer(strView); numTokens++; } + numColumns = std::max(numColumns, numTokens); } @@ -152,15 +157,17 @@ EncodeHelper(const std::vector& input, } template -template +template void StringEncoding:: EncodeHelper(const std::vector& input, - std::vector>& output, + std::vector>& output, const TokenizerType& tokenizer, PolicyType& policy, typename std::enable_if::onePassEncoding>::type*) { + policy.Reset(); + // The loop below extracts the tokens and writes the encoded values // at once. for (size_t i = 0; i < input.size(); i++) diff --git a/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt b/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt index 8a8c7ab65c..9f570e08d7 100644 --- a/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt +++ b/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt @@ -1,8 +1,10 @@ # Define the files that we need to compile. # Anything not in this list will not be compiled into mlpack. set(SOURCES + bag_of_words_encoding_policy.hpp dictionary_encoding_policy.hpp policy_traits.hpp + tf_idf_encoding_policy.hpp ) # add directory name to sources diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp new file mode 100644 index 0000000000..d71a82ac6e --- /dev/null +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -0,0 +1,171 @@ +/** + * @file bag_of_words_encoding_policy.hpp + * @author Jeffin Sam + * @author Mikhail Lozhnikov + * + * Definition of the BagOfWordsEncodingPolicy class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_DATA_STR_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_STR_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP + +#include +#include +#include + +namespace mlpack { +namespace data { + +/** + * Definition of the BagOfWordsEncodingPolicy class. + * + * BagOfWords is used as a helper class for StringEncoding. The encoder maps + * each dataset item to a vector of size N, where N is equal to the total unique + * number of tokens. The i-th coordinate of the output vector is equal to + * the number of times when the i-th token occurs in the corresponding dataset + * item. The order in which the tokens are labeled is defined by the dictionary + * used by the StringEncoding class. The encoder writes data either in the + * column-major order or in the row-major order depending on the output data + * type. + */ +class BagOfWordsEncodingPolicy +{ + public: + /** + * Clear the necessary internal variables. + */ + static void Reset() + { + // Nothing to do. + } + + /** + * The function initializes the output matrix. The encoder writes data + * in the column-major order. + * + * @tparam MatType The output matrix type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + * input dataset (not used). + * @param dictionarySize The size of the dictionary. + */ + template + static void InitMatrix(MatType& output, + const size_t datasetSize, + const size_t /* maxNumTokens */, + const size_t dictionarySize) + { + output.zeros(dictionarySize, datasetSize); + } + + /** + * The function initializes the output matrix. The encoder writes data + * in the row-major order. + * + * Overloaded function to save the result in vector>. + * + * @tparam ElemType Type of the output values. + * + * @param output Output matrix to store the encoded results. + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + * input dataset (not used). + * @param dictionarySize The size of the dictionary. + */ + template + static void InitMatrix(std::vector>& output, + const size_t datasetSize, + const size_t /* maxNumTokens */, + const size_t dictionarySize) + { + output.resize(datasetSize, std::vector(dictionarySize)); + } + + /** + * The function performs the bag of words encoding algorithm i.e. it writes + * the encoded token to the output. The encoder writes data in the + * column-major order. + * + * @tparam MatType The output matrix type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param line The line number at which the encoding is performed. + * @param index The token index in the line. + */ + template + static void Encode(MatType& output, + const size_t value, + const size_t line, + const size_t /* index */) + { + // The labels are assigned sequentially starting from one. + output(value - 1, line) += 1; + } + + /** + * The function performs the bag of words encoding algorithm i.e. it writes + * the encoded token to the output. The encoder writes data in the + * row-major order. + * + * Overloaded function to accept vector> as the output + * type. + * + * @tparam ElemType Type of the output values. + * + * @param output Output matrix to store the encoded results. + * @param value The encoded token. + * @param line The line number at which the encoding is performed. + * @param index The line token number at which the encoding is performed. + */ + template + static void Encode(std::vector>& output, + const size_t value, + const size_t line, + const size_t /* index */) + { + // The labels are assigned sequentially starting from one. + output[line][value - 1] += 1; + } + + /** + * The function is not used by the bag of words encoding policy. + * + * @param line The line number at which the encoding is performed. + * @param index The token sequence number in the line. + * @param value The encoded token. + */ + static void PreprocessToken(size_t /* line */, + size_t /* index */, + size_t /* value */) + { } + + /** + * Serialize the class to the given archive. + */ + template + void serialize(Archive& /* ar */, const unsigned int /* version */) + { + // Nothing to serialize. + } +}; + +/** + * A convenient alias for the StringEncoding class with BagOfWordsEncodingPolicy + * and the default dictionary for the given token type. + * + * @tparam TokenType Type of the tokens. + */ +template +using BagOfWordsEncoding = StringEncoding>; +} // namespace data +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index 1e67c6d29f..d9a37cacac 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -25,65 +25,92 @@ namespace data { * The encoder assigns a positive integer number to each unique token and treats * the dataset as categorical. The numbers are assigned sequentially starting * from one. The order in which the tokens are labeled is defined by - * the dictionary used by the StringEncoding class. + * the dictionary used by the StringEncoding class. The encoder writes data + * either in the column-major order or in the row-major order depending on + * the output data type. */ class DictionaryEncodingPolicy { public: /** - * The function initializes the output matrix. - * - * @tparam MatType The output matrix type. - * - * @param output Output matrix to store the encoded results (sp_mat or mat). - * @param datasetSize The number of strings in the input dataset. - * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset. - * @param dictionarySize The size of the dictionary (not used). - */ + * Clear the necessary internal variables. + */ + static void Reset() + { + // Nothing to do. + } + + /** + * The function initializes the output matrix. The encoder writes data + * in the column-major order. + * + * @tparam MatType The output matrix type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + * input dataset. + * @param dictionarySize The size of the dictionary (not used). + */ template static void InitMatrix(MatType& output, const size_t datasetSize, const size_t maxNumTokens, - const size_t /*dictionarySize*/) + const size_t /* dictionarySize */) { - output.zeros(datasetSize, maxNumTokens); + output.zeros(maxNumTokens, datasetSize); } - /** - * The function performs the dictionary encoding algorithm i.e. it writes - * the encoded token to the ouput. - * - * @tparam MatType The output matrix type. - * - * @param output Output matrix to store the encoded results (sp_mat or mat). - * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The token index in the row. - */ + /** + * The function performs the dictionary encoding algorithm i.e. it writes + * the encoded token to the output. The encoder writes data in the + * column-major order. + * + * @tparam MatType The output matrix type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param line The line number at which the encoding is performed. + * @param index The token index in the line. + */ template static void Encode(MatType& output, const size_t value, - const size_t row, - const size_t col) + const size_t line, + const size_t index) { - output(row, col) = value; + output(index, line) = value; } - /** + /** * The function performs the dictionary encoding algorithm i.e. it writes - * the encoded token to the ouput. This is an overload function which saves - * the result into the given vector to avoid padding. + * the encoded token to the output. This is an overloaded function which saves + * the result into the given vector to avoid padding. The encoder writes data + * in the row-major order. * - * @param output Output vector to store the encoded results. + * @tparam ElemType Type of the output values. + * + * @param output Output vector to store the encoded line. * @param value The encoded token. */ - static void Encode(std::vector& output, - const size_t value) + template + static void Encode(std::vector& output, size_t value) { output.push_back(value); } + /** + * The function is not used by the dictionary encoding policy. + * + * @param line The line number at which the encoding is performed. + * @param index The token sequence number in the line. + * @param value The encoded token. + */ + static void PreprocessToken(const size_t /* line */, + const size_t /* index */, + const size_t /* value */) + { } + /** * Serialize the class to the given archive. */ diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp new file mode 100644 index 0000000000..853493e285 --- /dev/null +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -0,0 +1,349 @@ +/** + * @file tf_idf_encoding_policy.hpp + * @author Jeffin Sam + * @author Mikhail Lozhnikov + * + * Definition of the TfIdfEncodingPolicy class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_DATA_STRING_ENCODING_POLICIES_TF_IDF_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_STRING_ENCODING_POLICIES_TF_IDF_ENCODING_POLICY_HPP + +#include +#include +#include + +namespace mlpack { +namespace data { + +/** + * Definition of the TfIdfEncodingPolicy class. TfIdfEncodingPolicy is used + * as a helper class for StringEncoding. + * + * Tf-idf is a weighting scheme that takes into account the importance of + * encoded tokens. The tf-idf statistics is equal to term frequency (tf) + * multiplied by inverse document frequency (idf). + * The encoder assigns the corresponding tf-idf value to each token. The order + * in which the tokens are labeled is defined by the dictionary used by the + * StringEncoding class. The encoder writes data either in the column-major + * order or in the row-major order depending on the output data type. + */ +class TfIdfEncodingPolicy +{ + public: + /** + * Enum class used to identify the type of the term frequency statistics. + * + * The present implementation supports the following types: + * BINARY Term frequency equals 1 if the row contains the encoded + * token and 0 otherwise. + * RAW_COUNT Term frequency equals the number of times when the encoded + * token occurs in the row. + * TERM_FREQUENCY Term frequency equals the number of times when the encoded + * token occurs in the row divided by the total number of + * tokens in the row. + * SUBLINEAR_TF Term frequency equals \f$ 1 + log(rawCount), \f$ where + * rawCount is equal to the number of times when the encoded + * token occurs in the row. + */ + enum class TfTypes + { + BINARY, + RAW_COUNT, + TERM_FREQUENCY, + SUBLINEAR_TF, + }; + + /** + * Construct this using the term frequency type and the inverse document + * frequency type. + * + * @param tfType Type of the term frequency statistics. + * @param smoothIdf Used to indicate whether to use smooth idf or not. + * If idf is smooth it's calculated by the following formula: + * \f$ idf(T) = \log \frac{1 + N}{1 + df(T)} + 1, \f$ where + * \f$ N \f$ is the total number of strings in the document, + * \f$ T \f$ is the current encoded token, \f$ df(T) \f$ + * equals the number of strings which contain the token. + * If idf isn't smooth then the following rule applies: + * \f$ idf(T) = \log \frac{N}{df(T)} + 1. \f$ + */ + TfIdfEncodingPolicy(const TfTypes tfType = TfTypes::RAW_COUNT, + const bool smoothIdf = true) : + tfType(tfType), + smoothIdf(smoothIdf) + { } + + /** + * Clear the necessary internal variables. + */ + void Reset() + { + tokensFrequences.clear(); + numContainingStrings.clear(); + linesSizes.clear(); + } + + /** + * The function initializes the output matrix. The encoder writes data + * in the row-major order. + * + * @tparam MatType The output matrix type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + * input dataset (not used). + * @param dictionarySize The size of the dictionary. + */ + template + static void InitMatrix(MatType& output, + const size_t datasetSize, + const size_t /* maxNumTokens */, + const size_t dictionarySize) + { + output.zeros(dictionarySize, datasetSize); + } + + /** + * The function initializes the output matrix. The encoder writes data + * in the row-major order. + * + * Overloaded function to save the result in vector>. + * + * @tparam ElemType Type of the output values. + * + * @param output Output matrix to store the encoded results. + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + * input dataset (not used). + * @param dictionarySize The size of the dictionary. + */ + template + static void InitMatrix(std::vector>& output, + const size_t datasetSize, + const size_t /* maxNumTokens */, + const size_t dictionarySize) + { + output.resize(datasetSize, std::vector(dictionarySize)); + } + + /** + * The function performs the TfIdf encoding algorithm i.e. it writes + * the encoded token to the output. The encoder writes data in the + * column-major order. + * + * @tparam MatType The output matrix type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param line The line number at which the encoding is performed. + * @param index The token index in the line. + */ + template + void Encode(MatType& output, + const size_t value, + const size_t line, + const size_t /* index */) + { + const typename MatType::elem_type tf = + TermFrequency( + tokensFrequences[line][value], linesSizes[line]); + + const typename MatType::elem_type idf = + InverseDocumentFrequency( + output.n_cols, numContainingStrings[value]); + + output(value - 1, line) = tf * idf; + } + + /** + * The function performs the TfIdf encoding algorithm i.e. it writes + * the encoded token to the output. The encoder writes data in the + * row-major order. + * + * Overloaded function to accept vector> as the output + * type. + * + * @tparam ElemType Type of the output values. + * + * @param output Output matrix to store the encoded results. + * @param value The encoded token. + * @param line The line number at which the encoding is performed. + * @param index The token index in the line. + */ + template + void Encode(std::vector>& output, + const size_t value, + const size_t line, + const size_t /* index */) + { + const ElemType tf = TermFrequency( + tokensFrequences[line][value], linesSizes[line]); + + const ElemType idf = InverseDocumentFrequency( + output.size(), numContainingStrings[value]); + + output[line][value - 1] = tf * idf; + } + + /* + * The function calculates the necessary statistics for the purpose + * of the tf-idf algorithm during the first pass through the dataset. + * + * @param line The line number at which the encoding is performed. + * @param index The token sequence number in the line. + * @param value The encoded token. + */ + void PreprocessToken(const size_t line, + const size_t /* index */, + const size_t value) + { + if (line >= tokensFrequences.size()) + { + linesSizes.resize(line + 1); + tokensFrequences.resize(line + 1); + } + + tokensFrequences[line][value]++; + + if (tokensFrequences[line][value] == 1) + numContainingStrings[value]++; + + linesSizes[line]++; + } + + //! Return token frequencies. + const std::vector>& + TokensFrequences() const { return tokensFrequences; } + //! Modify token frequencies. + std::vector>& TokensFrequences() + { + return tokensFrequences; + } + + //! Get the number of containing strings depending on the given token. + const std::unordered_map& NumContainingStrings() const + { + return numContainingStrings; + } + + //! Modify the number of containing strings depending on the given token. + std::unordered_map& NumContainingStrings() + { + return numContainingStrings; + } + + //! Return the lines sizes. + const std::vector& LinesSizes() const { return linesSizes; } + //! Modify the lines sizes. + std::vector& LinesSizes() { return linesSizes; } + + //! Return the term frequency type. + TfTypes TfType() const { return tfType; } + //! Modify the term frequency type. + TfTypes& TfType() { return tfType; } + + //! Determine the idf algorithm type (whether it's smooth or not). + bool SmoothIdf() const { return smoothIdf; } + //! Modify the idf algorithm type (whether it's smooth or not). + bool& SmoothIdf() { return smoothIdf; } + + /** + * Serialize the class to the given archive. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(tfType); + ar & BOOST_SERIALIZATION_NVP(smoothIdf); + } + + private: + /** + * The function calculates the term frequency statistics. + * + * @tparam ValueType Type of the returned value. + * + * @param numOccurrences The number of the given token occurrences in + * the line. + * @param numTokens The total number of tokens in the line. + */ + template + ValueType TermFrequency(const size_t numOccurrences, + const size_t numTokens) + { + switch (tfType) + { + case TfTypes::BINARY: + return numOccurrences > 0; + case TfTypes::RAW_COUNT: + return numOccurrences; + case TfTypes::TERM_FREQUENCY: + return static_cast(numOccurrences) / numTokens; + case TfTypes::SUBLINEAR_TF: + return std::log(static_cast(numOccurrences)) + 1; + default: + Log::Fatal << "Incorrect term frequency type!"; + return 0; + } + } + + /** + * The function calculates the inverse document frequency statistics. + * + * @tparam ValueType Type of the returned value. + * + * @param totalNumLines The total number of strings in the input dataset. + * @param numOccurrences The number of strings in the input dataset + * which contain the current token. + */ + template + ValueType InverseDocumentFrequency(const size_t totalNumLines, + const size_t numOccurrences) + { + if (smoothIdf) + { + return std::log(static_cast(totalNumLines + 1) / + (1 + numOccurrences)) + 1.0; + } + else + { + return std::log(static_cast(totalNumLines) / + numOccurrences) + 1.0; + } + } + + private: + //! Used to store the total number of tokens for each line. + std::vector> tokensFrequences; + /** + * Used to store the number of strings which contain a token depending + * on the given token. + */ + std::unordered_map numContainingStrings; + //! Used to store the number of tokens in each line. + std::vector linesSizes; + //! Type of the term frequency scheme. + TfTypes tfType; + //! Indicates whether the idf scheme is smooth or not. + bool smoothIdf; +}; + +/** + * A convenient alias for the StringEncoding class with TfIdfEncodingPolicy + * and the default dictionary for the given token type. + * + * @tparam TokenType Type of the tokens. + */ +template +using TfIdfEncoding = StringEncoding>; +} // namespace data +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/dists/gamma_distribution.cpp b/src/mlpack/core/dists/gamma_distribution.cpp index 54097c1fd0..63f7d7e29c 100644 --- a/src/mlpack/core/dists/gamma_distribution.cpp +++ b/src/mlpack/core/dists/gamma_distribution.cpp @@ -11,8 +11,8 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include "gamma_distribution.hpp" -// This will include digamma and trigamma. -#include +#include +#include using namespace mlpack; using namespace mlpack::distribution; diff --git a/src/mlpack/core/metrics/ip_metric_impl.hpp b/src/mlpack/core/metrics/ip_metric_impl.hpp index 777315a576..4c9091749c 100644 --- a/src/mlpack/core/metrics/ip_metric_impl.hpp +++ b/src/mlpack/core/metrics/ip_metric_impl.hpp @@ -49,7 +49,7 @@ IPMetric::~IPMetric() template IPMetric::IPMetric(const IPMetric& other) : - kernel(other.kernel), + kernel(!other.kernelOwner ? other.kernel : new KernelType(*other.kernel)), kernelOwner(other.kernelOwner) { // Nothing to do. @@ -90,7 +90,11 @@ void IPMetric::serialize(Archive& ar, // If we're loading, we need to allocate space for the kernel, and we will own // the kernel. if (Archive::is_loading::value) + { + if (kernelOwner) + delete kernel; kernelOwner = true; + } ar & BOOST_SERIALIZATION_NVP(kernel); } diff --git a/src/mlpack/core/metrics/lmetric.hpp b/src/mlpack/core/metrics/lmetric.hpp index d92bc0ea8f..370ecc7517 100644 --- a/src/mlpack/core/metrics/lmetric.hpp +++ b/src/mlpack/core/metrics/lmetric.hpp @@ -63,7 +63,7 @@ template class LMetric { public: - /*** + /** * Default constructor does nothing, but is required to satisfy the Metric * policy. */ diff --git a/src/mlpack/core/metrics/mahalanobis_distance.hpp b/src/mlpack/core/metrics/mahalanobis_distance.hpp index 3547202eee..9c251cbeb3 100644 --- a/src/mlpack/core/metrics/mahalanobis_distance.hpp +++ b/src/mlpack/core/metrics/mahalanobis_distance.hpp @@ -1,4 +1,4 @@ -/*** +/** * @file mahalanobis_distance.hpp * @author Ryan Curtin * diff --git a/src/mlpack/core/metrics/mahalanobis_distance_impl.hpp b/src/mlpack/core/metrics/mahalanobis_distance_impl.hpp index 58aea1ab5d..16763cc4c5 100644 --- a/src/mlpack/core/metrics/mahalanobis_distance_impl.hpp +++ b/src/mlpack/core/metrics/mahalanobis_distance_impl.hpp @@ -1,4 +1,4 @@ -/*** +/** * @file mahalanobis_distance_impl.hpp * @author Ryan Curtin * diff --git a/src/mlpack/core/tree/hollow_ball_bound_impl.hpp b/src/mlpack/core/tree/hollow_ball_bound_impl.hpp index e6d9737ee6..88e3e082ea 100644 --- a/src/mlpack/core/tree/hollow_ball_bound_impl.hpp +++ b/src/mlpack/core/tree/hollow_ball_bound_impl.hpp @@ -80,6 +80,9 @@ template HollowBallBound& HollowBallBound:: operator=(const HollowBallBound& other) { + if (ownsMetric) + delete metric; + radii = other.radii; center = other.center; hollowCenter = other.hollowCenter; diff --git a/src/mlpack/core/util/prefixedoutstream.hpp b/src/mlpack/core/util/prefixedoutstream.hpp index 6078edde1b..138a79109d 100644 --- a/src/mlpack/core/util/prefixedoutstream.hpp +++ b/src/mlpack/core/util/prefixedoutstream.hpp @@ -124,7 +124,7 @@ class PrefixedOutStream private: /** * Conducts the base logic required in all the operator << overloads. Mostly - * just a good idea to reduce copy-pasta. + * just a good idea to reduce copy-paste. * * This overload is for non-Armadillo objects, which need special handling * during printing. @@ -138,7 +138,7 @@ class PrefixedOutStream /** * Conducts the base logic required in all the operator << overloads. Mostly - * just a good idea to reduce copy-pasta. + * just a good idea to reduce copy-paste. * * This overload is for Armadillo objects, which need special handling during * printing. diff --git a/src/mlpack/methods/ann/brnn.hpp b/src/mlpack/methods/ann/brnn.hpp index cddc9c8544..7e042de6fd 100644 --- a/src/mlpack/methods/ann/brnn.hpp +++ b/src/mlpack/methods/ann/brnn.hpp @@ -24,6 +24,7 @@ #include "init_rules/network_init.hpp" #include #include +#include #include #include @@ -72,10 +73,41 @@ class BRNN BRNN(const size_t rho, const bool single = false, OutputLayerType outputLayer = OutputLayerType(), - MergeLayerType mergeLayer = MergeLayerType(), - MergeOutputType mergeOutput = MergeOutputType(), + MergeLayerType* mergeLayer = new MergeLayerType(), + MergeOutputType* mergeOutput = new MergeOutputType(), InitializationRuleType initializeRule = InitializationRuleType()); + ~BRNN(); + + /** + * Check if the optimizer has MaxIterations() parameter, if it does + * then check if it's value is less than the number of datapoints + * in the dataset. + * + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. + */ + template + typename std::enable_if< + HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; + + /** + * Check if the optimizer has MaxIterations() parameter, if it + * doesn't then simply return from the function. + * + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. + */ + template + typename std::enable_if< + !HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; + /** * Train the bidirectional recurrent neural network on the given input data * using the given optimizer. diff --git a/src/mlpack/methods/ann/brnn_impl.hpp b/src/mlpack/methods/ann/brnn_impl.hpp index 4778204cef..f7ceef7677 100644 --- a/src/mlpack/methods/ann/brnn_impl.hpp +++ b/src/mlpack/methods/ann/brnn_impl.hpp @@ -39,13 +39,13 @@ BRNN +BRNN::~BRNN() +{ + // Remove the last layers from the forward and backward RNNs, as they are held + // in mergeLayer. So, when we use DeleteVisitor with mergeLayer, those two + // layers will be properly (and not doubly) freed. + forwardRNN.network.pop_back(); + backwardRNN.network.pop_back(); + + // Clean up layers that we allocated. + boost::apply_visitor(DeleteVisitor(), mergeLayer); + boost::apply_visitor(DeleteVisitor(), mergeOutput); +} + +template +template +typename std::enable_if< + HasMaxIterations + ::value, void>::type +BRNN::WarnMessageMaxIterations +(OptimizerType& optimizer, size_t samples) const +{ + if (optimizer.MaxIterations() < samples && + optimizer.MaxIterations() != 0) + { + Log::Warn << "The optimizer's maximum number of iterations " + << "is less than the size of the dataset; the " + << "optimizer will not pass over the entire " + << "dataset. To fix this, modify the maximum " + << "number of iterations to be at least equal " + << "to the number of points of your dataset " + << "(" << samples << ")." << std::endl; + } +} + +template +template +typename std::enable_if< + !HasMaxIterations + ::value, void>::type +BRNN::WarnMessageMaxIterations +(OptimizerType& optimizer, size_t samples) const +{ + return; +} + template @@ -83,6 +138,8 @@ double BRNN(optimizer, this->predictors.n_cols); + // Train the model. Timer::Start("BRNN_optimization"); const double out = optimizer.Optimize(*this, parameter); @@ -117,6 +174,8 @@ double BRNN(optimizer, this->predictors.n_cols); + // Train the model. const double out = optimizer.Optimize(*this, parameter); @@ -166,34 +225,34 @@ void BRNN results1, results2; for (size_t seqNum = 0; seqNum < rho; ++seqNum) { - forwardRNN.Forward(std::move(arma::mat( + forwardRNN.Forward(arma::mat( predictors.slice(seqNum).colptr(begin), - predictors.n_rows, batchSize, false, true))); - backwardRNN.Forward(std::move(arma::mat( + predictors.n_rows, batchSize, false, true)); + backwardRNN.Forward(arma::mat( predictors.slice(rho - seqNum - 1).colptr(begin), - predictors.n_rows, batchSize, false, true))); + predictors.n_rows, batchSize, false, true)); - boost::apply_visitor(SaveOutputParameterVisitor( - std::move(results1)), forwardRNN.network.back()); - boost::apply_visitor(SaveOutputParameterVisitor( - std::move(results2)), backwardRNN.network.back()); + boost::apply_visitor(SaveOutputParameterVisitor(results1), + forwardRNN.network.back()); + boost::apply_visitor(SaveOutputParameterVisitor(results2), + backwardRNN.network.back()); } if (outputSize == 0) { @@ -271,22 +330,22 @@ double BRNN results1, results2; for (size_t seqNum = 0; seqNum < rho; ++seqNum) { - forwardRNN.Forward(std::move(arma::mat( + forwardRNN.Forward(arma::mat( predictors.slice(seqNum).colptr(begin), - predictors.n_rows, batchSize, false, true))); - backwardRNN.Forward(std::move(arma::mat( + predictors.n_rows, batchSize, false, true)); + backwardRNN.Forward(arma::mat( predictors.slice(rho - seqNum - 1).colptr(begin), - predictors.n_rows, batchSize, false, true))); + predictors.n_rows, batchSize, false, true)); for (size_t l = 0; l < networkSize; ++l) { boost::apply_visitor(SaveOutputParameterVisitor( - std::move(forwardRNNOutputParameter)), forwardRNN.network[l]); + forwardRNNOutputParameter), forwardRNN.network[l]); boost::apply_visitor(SaveOutputParameterVisitor( - std::move(backwardRNNOutputParameter)), backwardRNN.network[l]); + backwardRNNOutputParameter), backwardRNN.network[l]); } - boost::apply_visitor(SaveOutputParameterVisitor( - std::move(results1)), forwardRNN.network.back()); - boost::apply_visitor(SaveOutputParameterVisitor( - std::move(results2)), backwardRNN.network.back()); + boost::apply_visitor(SaveOutputParameterVisitor(results1), + forwardRNN.network.back()); + boost::apply_visitor(SaveOutputParameterVisitor(results2), + backwardRNN.network.back()); } if (outputSize == 0) { @@ -410,18 +469,18 @@ EvaluateWithGradient(const arma::mat& /* parameters */, responseSeq = seqNum; } boost::apply_visitor(LoadOutputParameterVisitor( - std::move(results1)), forwardRNN.network.back()); + results1), forwardRNN.network.back()); boost::apply_visitor(LoadOutputParameterVisitor( - std::move(results2)), backwardRNN.network.back()); - boost::apply_visitor(ForwardVisitor(std::move(input), - std::move(boost::apply_visitor(outputParameterVisitor, mergeLayer))), + results2), backwardRNN.network.back()); + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, mergeLayer)), mergeLayer); boost::apply_visitor(ForwardVisitor( - std::move(boost::apply_visitor(outputParameterVisitor, mergeLayer)), - std::move(results.slice(seqNum))), mergeOutput); - performance += outputLayer.Forward(std::move(results.slice(seqNum)), - std::move(arma::mat(responses.slice(responseSeq).colptr(begin), - responses.n_rows, batchSize, false, true))); + boost::apply_visitor(outputParameterVisitor, mergeLayer), + results.slice(seqNum)), mergeOutput); + performance += outputLayer.Forward(results.slice(seqNum), + arma::mat(responses.slice(responseSeq).colptr(begin), + responses.n_rows, batchSize, false, true)); } // Calculate and storing delta parameters from output for t = 1 to T. @@ -436,25 +495,25 @@ EvaluateWithGradient(const arma::mat& /* parameters */, } else if (single && seqNum == 0) { - outputLayer.Backward(std::move(results.slice(seqNum)), - std::move(arma::mat(responses.slice(0).colptr(begin), - responses.n_rows, batchSize, false, true)), std::move(error)); + outputLayer.Backward(results.slice(seqNum), + arma::mat(responses.slice(0).colptr(begin), + responses.n_rows, batchSize, false, true), error); } else { - outputLayer.Backward(std::move(results.slice(seqNum)), - std::move(arma::mat(responses.slice(seqNum).colptr(begin), - responses.n_rows, batchSize, false, true)), std::move(error)); + outputLayer.Backward(results.slice(seqNum), + arma::mat(responses.slice(seqNum).colptr(begin), + responses.n_rows, batchSize, false, true), error); } - boost::apply_visitor(BackwardVisitor(std::move(results.slice(seqNum)), - std::move(error), std::move(delta)), mergeOutput); + boost::apply_visitor(BackwardVisitor(results.slice(seqNum), error, delta), + mergeOutput); allDelta.push_back(arma::mat(delta)); } // BPTT ForwardRNN from t = T to 1. totalGradient = arma::mat(gradient.memptr(), - parameter.n_elem/2, 1, false, false); + parameter.n_elem / 2, 1, false, false); forwardGradient.zeros(); forwardRNN.ResetGradients(forwardGradient); @@ -467,32 +526,32 @@ EvaluateWithGradient(const arma::mat& /* parameters */, for (size_t l = 0; l < networkSize; ++l) { boost::apply_visitor(LoadOutputParameterVisitor( - std::move(forwardRNNOutputParameter)), + forwardRNNOutputParameter), forwardRNN.network[networkSize - 1 - l]); } - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, forwardRNN.network.back())), - std::move(allDelta[rho - seqNum - 1]), std::move(delta), 0), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, forwardRNN.network.back()), + allDelta[rho - seqNum - 1], delta, 0), mergeLayer); for (size_t i = 2; i < networkSize; ++i) { boost::apply_visitor(BackwardVisitor( - std::move(boost::apply_visitor(outputParameterVisitor, - forwardRNN.network[networkSize - i])), - std::move(boost::apply_visitor(deltaVisitor, - forwardRNN.network[networkSize - i + 1])), std::move( + boost::apply_visitor(outputParameterVisitor, + forwardRNN.network[networkSize - i]), boost::apply_visitor(deltaVisitor, - forwardRNN.network[networkSize - i]))), + forwardRNN.network[networkSize - i + 1]), + boost::apply_visitor(deltaVisitor, + forwardRNN.network[networkSize - i])), forwardRNN.network[networkSize - i]); } - forwardRNN.Gradient(std::move( + forwardRNN.Gradient( arma::mat(predictors.slice(rho - seqNum - 1).colptr(begin), - predictors.n_rows, batchSize, false, true))); + predictors.n_rows, batchSize, false, true)); boost::apply_visitor(GradientVisitor( - std::move(boost::apply_visitor(outputParameterVisitor, - forwardRNN.network[networkSize - 2])), - std::move(allDelta[rho - seqNum - 1]), 0), mergeLayer); + boost::apply_visitor(outputParameterVisitor, + forwardRNN.network[networkSize - 2]), + allDelta[rho - seqNum - 1], 0), mergeLayer); totalGradient += forwardGradient; } @@ -506,31 +565,31 @@ EvaluateWithGradient(const arma::mat& /* parameters */, for (size_t l = 0; l < networkSize; ++l) { boost::apply_visitor(LoadOutputParameterVisitor( - std::move(backwardRNNOutputParameter)), + backwardRNNOutputParameter), backwardRNN.network[networkSize - 1 - l]); } - boost::apply_visitor(BackwardVisitor(std::move( + boost::apply_visitor(BackwardVisitor( boost::apply_visitor(outputParameterVisitor, - backwardRNN.network.back())), - std::move(allDelta[seqNum]), std::move(delta), 1), mergeLayer); + backwardRNN.network.back()), + allDelta[seqNum], delta, 1), mergeLayer); for (size_t i = 2; i < networkSize; ++i) { boost::apply_visitor(BackwardVisitor( - std::move(boost::apply_visitor(outputParameterVisitor, - backwardRNN.network[networkSize - i])), std::move(boost::apply_visitor( - deltaVisitor, backwardRNN.network[networkSize - i + 1])), std::move( + boost::apply_visitor(outputParameterVisitor, + backwardRNN.network[networkSize - i]), boost::apply_visitor( + deltaVisitor, backwardRNN.network[networkSize - i + 1]), boost::apply_visitor(deltaVisitor, - backwardRNN.network[networkSize - i]))), + backwardRNN.network[networkSize - i])), backwardRNN.network[networkSize - i]); } - backwardRNN.Gradient(std::move( + backwardRNN.Gradient( arma::mat(predictors.slice(seqNum).colptr(begin), - predictors.n_rows, batchSize, false, true))); + predictors.n_rows, batchSize, false, true)); boost::apply_visitor(GradientVisitor( std::move(boost::apply_visitor(outputParameterVisitor, backwardRNN.network[networkSize - 2])), - std::move(allDelta[seqNum]), 1), mergeLayer); + allDelta[seqNum], 1), mergeLayer); totalGradient += backwardGradient; } return performance; @@ -592,6 +651,8 @@ void BRNN( forwardRNN.network.back()), mergeLayer); boost::apply_visitor(AddVisitor( @@ -665,6 +726,8 @@ void BRNN::BernoulliDistribution() : template BernoulliDistribution::BernoulliDistribution( - const DataType&& param, + const DataType& param, const bool applyLogistic, const double eps) : logits(param), @@ -36,7 +36,9 @@ BernoulliDistribution::BernoulliDistribution( eps(eps) { if (applyLogistic) + { LogisticFunction::Fn(logits, probability); + } else { probability = arma::mat(logits.memptr(), logits.n_rows, @@ -58,7 +60,7 @@ DataType BernoulliDistribution::Sample() const template double BernoulliDistribution::LogProbability( - const DataType&& observation) const + const DataType& observation) const { return arma::accu(arma::log(probability + eps) % observation + arma::log(1 - probability + eps) % (1 - observation)) / @@ -67,7 +69,7 @@ double BernoulliDistribution::LogProbability( template void BernoulliDistribution::LogProbBackward( - const DataType&& observation, DataType&& output) const + const DataType& observation, DataType& output) const { if (!applyLogistic) { diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index a3b0442dcd..70875f2149 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include namespace mlpack { @@ -82,6 +83,35 @@ class FFN //! Destructor to release allocated memory. ~FFN(); + /** + * Check if the optimizer has MaxIterations() parameter, if it does + * then check if it's value is less than the number of datapoints + * in the dataset. + * + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. + */ + template + typename std::enable_if< + HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; + + /** + * Check if the optimizer has MaxIterations() parameter, if it + * doesn't then simply return from the function. + * + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. + */ + template + typename std::enable_if< + !HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; + /** * Train the feedforward network on the given input data using the given * optimizer. @@ -153,7 +183,9 @@ class FFN * @param predictors Input variables. * @param responses Target outputs for input variables. */ - double Evaluate(arma::mat predictors, arma::mat responses); + template + double Evaluate(const PredictorsType& predictors, + const ResponsesType& responses); /** * Evaluate the feedforward network with the given parameters. This function @@ -313,7 +345,8 @@ class FFN * @param inputs The input data. * @param results The predicted results. */ - void Forward(arma::mat inputs, arma::mat& results); + template + void Forward(const PredictorsType& inputs, ResponsesType& results); /** * Perform a partial forward pass of the data. @@ -326,8 +359,9 @@ class FFN * @param begin The index of the first layer. * @param end The index of the last layer. */ - void Forward(arma::mat inputs, - arma::mat& results, + template + void Forward(const PredictorsType& inputs , + ResponsesType& results, const size_t begin, const size_t end); @@ -342,7 +376,12 @@ class FFN * @param gradients Computed gradients. * @return Training error of the current pass. */ - double Backward(arma::mat targets, arma::mat& gradients); + template + double Backward(const PredictorsType& inputs, + const TargetsType& targets, + GradientsType& gradients); private: // Helper functions. @@ -352,7 +391,8 @@ class FFN * * @param input Data sequence to compute probabilities for. */ - void Forward(arma::mat&& input); + template + void Forward(const InputType& input); /** * Prepare the network for the given data. @@ -373,7 +413,8 @@ class FFN * Iterate through all layer modules and update the the gradient using the * layer defined optimizer. */ - void Gradient(arma::mat&& input); + template + void Gradient(const InputType& input); /** * Reset the module status by setting the current deterministic parameter @@ -427,9 +468,6 @@ class FFN //! The current error for the backward pass. arma::mat error; - //! THe current input of the forward/backward pass. - arma::mat currentInput; - //! Locally-stored delta visitor. DeltaVisitor deltaVisitor; @@ -496,7 +534,7 @@ template> { - BOOST_STATIC_CONSTANT(int, value = 1); + BOOST_STATIC_CONSTANT(int, value = 2); }; } // namespace serialization diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index f39bbfdb13..d2f71fcdef 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -41,7 +41,7 @@ FFN::FFN( numFunctions(0), deterministic(true) { - /* Nothing to do here */ + /* Nothing to do here. */ } template::ResetData( ResetParameters(); } +template +template +typename std::enable_if< + HasMaxIterations + ::value, void>::type +FFN:: +WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const +{ + if (optimizer.MaxIterations() < samples && + optimizer.MaxIterations() != 0) + { + Log::Warn << "The optimizer's maximum number of iterations " + << "is less than the size of the dataset; the " + << "optimizer will not pass over the entire " + << "dataset. To fix this, modify the maximum " + << "number of iterations to be at least equal " + << "to the number of points of your dataset " + << "(" << samples << ")." << std::endl; + } +} + +template +template +typename std::enable_if< + !HasMaxIterations + ::value, void>::type +FFN:: +WarnMessageMaxIterations(OptimizerType& /* optimizer */, size_t /* samples */) + const +{ + return; +} + template template @@ -78,6 +113,8 @@ double FFN::Train( { ResetData(std::move(predictors), std::move(responses)); + WarnMessageMaxIterations(optimizer, this->predictors.n_cols); + // Train the model. Timer::Start("ffn_optimization"); const double out = optimizer.Optimize(*this, parameter, callbacks...); @@ -100,6 +137,8 @@ double FFN::Train( OptimizerType optimizer; + WarnMessageMaxIterations(optimizer, this->predictors.n_cols); + // Train the model. Timer::Start("ffn_optimization"); const double out = optimizer.Optimize(*this, parameter, callbacks...); @@ -112,8 +151,9 @@ double FFN::Train( template +template void FFN::Forward( - arma::mat inputs, arma::mat& results) + const PredictorsType& inputs, ResponsesType& results) { if (parameter.is_empty()) ResetParameters(); @@ -124,25 +164,28 @@ void FFN::Forward( ResetDeterministic(); } - currentInput = std::move(inputs); - Forward(std::move(currentInput)); + Forward(inputs); results = boost::apply_visitor(outputParameterVisitor, network.back()); } template +template void FFN::Forward( - arma::mat inputs, arma::mat& results, const size_t begin, const size_t end) + const PredictorsType& inputs, + ResponsesType& results, + const size_t begin, + const size_t end) { - boost::apply_visitor(ForwardVisitor(std::move(inputs), std::move( - boost::apply_visitor(outputParameterVisitor, network[begin]))), + boost::apply_visitor(ForwardVisitor(inputs, + boost::apply_visitor(outputParameterVisitor, network[begin])), network[begin]); for (size_t i = 1; i < end - begin + 1; ++i) { - boost::apply_visitor(ForwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[begin + i - 1])), std::move( - boost::apply_visitor(outputParameterVisitor, network[begin + i]))), + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[begin + i - 1]), + boost::apply_visitor(outputParameterVisitor, network[begin + i])), network[begin + i]); } @@ -151,25 +194,28 @@ void FFN::Forward( template +template double FFN::Backward( - arma::mat targets, arma::mat& gradients) + const PredictorsType& inputs, + const TargetsType& targets, + GradientsType& gradients) { - double res = outputLayer.Forward(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), std::move(targets)); + double res = outputLayer.Forward(boost::apply_visitor( + outputParameterVisitor, network.back()), targets); for (size_t i = 0; i < network.size(); ++i) { res += boost::apply_visitor(lossVisitor, network[i]); } - outputLayer.Backward(std::move(boost::apply_visitor(outputParameterVisitor, - network.back())), std::move(targets), std::move(error)); + outputLayer.Backward(boost::apply_visitor(outputParameterVisitor, + network.back()), targets, error); gradients = arma::zeros(parameter.n_rows, parameter.n_cols); Backward(); ResetGradients(gradients); - Gradient(std::move(currentInput)); + Gradient(inputs); return res; } @@ -189,8 +235,7 @@ void FFN::Predict( } arma::mat resultsTemp; - Forward(std::move(arma::mat(predictors.colptr(0), - predictors.n_rows, 1, false, true))); + Forward(arma::mat(predictors.colptr(0), predictors.n_rows, 1, false, true)); resultsTemp = boost::apply_visitor(outputParameterVisitor, network.back()).col(0); @@ -199,8 +244,7 @@ void FFN::Predict( for (size_t i = 1; i < predictors.n_cols; i++) { - Forward(std::move(arma::mat(predictors.colptr(i), - predictors.n_rows, 1, false, true))); + Forward(arma::mat(predictors.colptr(i), predictors.n_rows, 1, false, true)); resultsTemp = boost::apply_visitor(outputParameterVisitor, network.back()); @@ -210,8 +254,9 @@ void FFN::Predict( template +template double FFN::Evaluate( - arma::mat predictors, arma::mat responses) + const PredictorsType& predictors, const ResponsesType& responses) { if (parameter.is_empty()) ResetParameters(); @@ -222,10 +267,10 @@ double FFN::Evaluate( ResetDeterministic(); } - Forward(std::move(predictors)); + Forward(predictors); - double res = outputLayer.Forward(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), std::move(responses)); + double res = outputLayer.Forward(boost::apply_visitor( + outputParameterVisitor, network.back()), responses); for (size_t i = 0; i < network.size(); ++i) { @@ -264,10 +309,10 @@ double FFN::Evaluate( ResetDeterministic(); } - Forward(std::move(predictors.cols(begin, begin + batchSize - 1))); + Forward(predictors.cols(begin, begin + batchSize - 1)); double res = outputLayer.Forward( - std::move(boost::apply_visitor(outputParameterVisitor, network.back())), - std::move(responses.cols(begin, begin + batchSize - 1))); + boost::apply_visitor(outputParameterVisitor, network.back()), + responses.cols(begin, begin + batchSize - 1)); for (size_t i = 0; i < network.size(); ++i) { @@ -325,10 +370,10 @@ EvaluateWithGradient(const arma::mat& /* parameters */, ResetDeterministic(); } - Forward(std::move(predictors.cols(begin, begin + batchSize - 1))); + Forward(predictors.cols(begin, begin + batchSize - 1)); double res = outputLayer.Forward( - std::move(boost::apply_visitor(outputParameterVisitor, network.back())), - std::move(responses.cols(begin, begin + batchSize - 1))); + boost::apply_visitor(outputParameterVisitor, network.back()), + responses.cols(begin, begin + batchSize - 1)); for (size_t i = 0; i < network.size(); ++i) { @@ -336,13 +381,13 @@ EvaluateWithGradient(const arma::mat& /* parameters */, } outputLayer.Backward( - std::move(boost::apply_visitor(outputParameterVisitor, network.back())), - std::move(responses.cols(begin, begin + batchSize - 1)), - std::move(error)); + boost::apply_visitor(outputParameterVisitor, network.back()), + responses.cols(begin, begin + batchSize - 1), + error); Backward(); ResetGradients(gradient); - Gradient(std::move(predictors.cols(begin, begin + batchSize - 1))); + Gradient(predictors.cols(begin, begin + batchSize - 1)); return res; } @@ -396,18 +441,19 @@ void FFN +template void FFN::Forward(arma::mat&& input) + CustomLayers...>::Forward(const InputType& input) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, network.front()))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network.front())), network.front()); if (!reset) @@ -434,9 +480,9 @@ void FFN void FFN::Backward() { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), std::move(error), std::move( - boost::apply_visitor(deltaVisitor, network.back()))), network.back()); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network.back()), error, + boost::apply_visitor(deltaVisitor, network.back())), network.back()); for (size_t i = 2; i < network.size(); ++i) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[network.size() - i])), std::move( - boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])), - std::move(boost::apply_visitor(deltaVisitor, - network[network.size() - i]))), network[network.size() - i]); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i])), + network[network.size() - i]); } } template +template void FFN::Gradient(arma::mat&& input) + CustomLayers...>::Gradient(const InputType& input) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move( - boost::apply_visitor(deltaVisitor, network[1]))), network.front()); + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, network[1])), network.front()); for (size_t i = 1; i < network.size() - 1; ++i) { - boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[i - 1])), std::move( - boost::apply_visitor(deltaVisitor, network[i + 1]))), network[i]); + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[i - 1]), + boost::apply_visitor(deltaVisitor, network[i + 1])), network[i]); } - boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[network.size() - 2])), std::move(error)), + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - 2]), error), network[network.size() - 1]); } @@ -505,7 +552,13 @@ void FFN::serialize( ar & BOOST_SERIALIZATION_NVP(parameter); ar & BOOST_SERIALIZATION_NVP(width); ar & BOOST_SERIALIZATION_NVP(height); - ar & BOOST_SERIALIZATION_NVP(currentInput); + + // Early versions used the currentInput member, which is now no longer needed. + if (version < 2) + { + arma::mat currentInput; // Temporary matrix to output. + ar & BOOST_SERIALIZATION_NVP(currentInput); + } // Earlier versions of the FFN code did not serialize whether or not the model // was reset. @@ -535,8 +588,8 @@ void FFN::serialize( size_t offset = 0; for (size_t i = 0; i < network.size(); ++i) { - offset += boost::apply_visitor(WeightSetVisitor(std::move(parameter), - offset), network[i]); + offset += boost::apply_visitor(WeightSetVisitor(parameter, offset), + network[i]); boost::apply_visitor(resetVisitor, network[i]); } @@ -562,7 +615,6 @@ void FFN::FFN( parameter(network.parameter), numFunctions(network.numFunctions), error(network.error), - currentInput(network.currentInput), deterministic(network.deterministic), delta(network.delta), inputParameter(network.inputParameter), @@ -613,7 +664,6 @@ FFN::FFN( parameter(std::move(network.parameter)), numFunctions(network.numFunctions), error(std::move(network.error)), - currentInput(std::move(network.currentInput)), deterministic(network.deterministic), delta(std::move(network.delta)), inputParameter(std::move(network.inputParameter)), diff --git a/src/mlpack/methods/ann/gan/gan.hpp b/src/mlpack/methods/ann/gan/gan.hpp index e90396c98b..b98c17f03c 100644 --- a/src/mlpack/methods/ann/gan/gan.hpp +++ b/src/mlpack/methods/ann/gan/gan.hpp @@ -289,7 +289,7 @@ class GAN * * @param input Sampled noise. */ - void Forward(arma::mat&& input); + void Forward(const arma::mat& input); /** * This function predicts the output of the network on the given input. diff --git a/src/mlpack/methods/ann/gan/gan_impl.hpp b/src/mlpack/methods/ann/gan/gan_impl.hpp index 9e45b63a30..6f622856b4 100644 --- a/src/mlpack/methods/ann/gan/gan_impl.hpp +++ b/src/mlpack/methods/ann/gan/gan_impl.hpp @@ -258,28 +258,27 @@ GAN::Evaluate( currentTarget = arma::mat(responses.memptr() + i, 1, batchSize, false, false); - discriminator.Forward(std::move(currentInput)); + discriminator.Forward(currentInput); double res = discriminator.outputLayer.Forward( - std::move(boost::apply_visitor( + boost::apply_visitor( outputParameterVisitor, - discriminator.network.back())), std::move(currentTarget)); + discriminator.network.back()), currentTarget); noise.imbue( [&]() { return noiseFunction();} ); - generator.Forward(std::move(noise)); + generator.Forward(noise); predictors.cols(numFunctions, numFunctions + batchSize - 1) = boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.Forward(std::move(predictors.cols(numFunctions, - numFunctions + batchSize - 1))); + discriminator.Forward(predictors.cols(numFunctions, + numFunctions + batchSize - 1)); responses.cols(numFunctions, numFunctions + batchSize - 1) = arma::zeros(1, batchSize); currentTarget = arma::mat(responses.memptr() + numFunctions, 1, batchSize, false, false); res += discriminator.outputLayer.Forward( - std::move(boost::apply_visitor( - outputParameterVisitor, - discriminator.network.back())), std::move(currentTarget)); + boost::apply_visitor(outputParameterVisitor, + discriminator.network.back()), currentTarget); return res; } @@ -341,7 +340,7 @@ EvaluateWithGradient(const arma::mat& /* parameters */, i, gradientDiscriminator, batchSize); noise.imbue( [&]() { return noiseFunction();} ); - generator.Forward(std::move(noise)); + generator.Forward(noise); predictors.cols(numFunctions, numFunctions + batchSize - 1) = boost::apply_visitor(outputParameterVisitor, generator.network.back()); responses.cols(numFunctions, numFunctions + batchSize - 1) = @@ -419,18 +418,18 @@ template< typename PolicyType > void GAN::Forward( - arma::mat&& input) + const arma::mat& input) { if (parameter.is_empty()) { Reset(); } - generator.Forward(std::move(input)); + generator.Forward(input); arma::mat ganOutput = boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.Forward(std::move(ganOutput)); + discriminator.Forward(ganOutput); } template< @@ -453,7 +452,7 @@ Predict(arma::mat input, arma::mat& output) ResetDeterministic(); } - Forward(std::move(input)); + Forward(input); output = boost::apply_visitor(outputParameterVisitor, discriminator.network.back()); @@ -502,8 +501,8 @@ serialize(Archive& ar, const unsigned int /* version */) size_t offset = 0; for (size_t i = 0; i < generator.network.size(); ++i) { - offset += boost::apply_visitor(WeightSetVisitor(std::move( - generator.parameter), offset), generator.network[i]); + offset += boost::apply_visitor(WeightSetVisitor( + generator.parameter, offset), generator.network[i]); boost::apply_visitor(resetVisitor, generator.network[i]); } @@ -511,8 +510,8 @@ serialize(Archive& ar, const unsigned int /* version */) offset = 0; for (size_t i = 0; i < discriminator.network.size(); ++i) { - offset += boost::apply_visitor(WeightSetVisitor(std::move( - discriminator.parameter), offset), discriminator.network[i]); + offset += boost::apply_visitor(WeightSetVisitor( + discriminator.parameter, offset), discriminator.network[i]); boost::apply_visitor(resetVisitor, discriminator.network[i]); } diff --git a/src/mlpack/methods/ann/gan/wgan_impl.hpp b/src/mlpack/methods/ann/gan/wgan_impl.hpp index 76073ef3b1..b21d685b68 100644 --- a/src/mlpack/methods/ann/gan/wgan_impl.hpp +++ b/src/mlpack/methods/ann/gan/wgan_impl.hpp @@ -50,28 +50,28 @@ GAN::Evaluate( currentTarget = arma::mat(responses.memptr() + i, 1, batchSize, false, false); - discriminator.Forward(std::move(currentInput)); + discriminator.Forward(currentInput); double res = discriminator.outputLayer.Forward( - std::move(boost::apply_visitor( + boost::apply_visitor( outputParameterVisitor, - discriminator.network.back())), std::move(currentTarget)); + discriminator.network.back()), currentTarget); noise.imbue( [&]() { return noiseFunction();} ); - generator.Forward(std::move(noise)); + generator.Forward(noise); predictors.cols(numFunctions, numFunctions + batchSize - 1) = boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.Forward(std::move(predictors.cols(numFunctions, - numFunctions + batchSize - 1))); + discriminator.Forward(predictors.cols(numFunctions, + numFunctions + batchSize - 1)); responses.cols(numFunctions, numFunctions + batchSize - 1) = -arma::ones(1, batchSize); currentTarget = arma::mat(responses.memptr() + numFunctions, 1, batchSize, false, false); res += discriminator.outputLayer.Forward( - std::move(boost::apply_visitor( + boost::apply_visitor( outputParameterVisitor, - discriminator.network.back())), std::move(currentTarget)); + discriminator.network.back()), currentTarget); return res; } @@ -132,7 +132,7 @@ EvaluateWithGradient(const arma::mat& /* parameters */, i, gradientDiscriminator, batchSize); noise.imbue( [&]() { return noiseFunction();} ); - generator.Forward(std::move(noise)); + generator.Forward(noise); predictors.cols(numFunctions, numFunctions + batchSize - 1) = boost::apply_visitor(outputParameterVisitor, generator.network.back()); responses.cols(numFunctions, numFunctions + batchSize - 1) = diff --git a/src/mlpack/methods/ann/init_rules/network_init.hpp b/src/mlpack/methods/ann/init_rules/network_init.hpp index 45232e3a67..68e75789e5 100644 --- a/src/mlpack/methods/ann/init_rules/network_init.hpp +++ b/src/mlpack/methods/ann/init_rules/network_init.hpp @@ -92,8 +92,8 @@ class NetworkInitialization // hold various other modules. for (size_t i = 0, offset = parameterOffset; i < network.size(); ++i) { - offset += boost::apply_visitor(WeightSetVisitor(std::move(parameter), - offset), network[i]); + offset += boost::apply_visitor(WeightSetVisitor(parameter, offset), + network[i]); boost::apply_visitor(resetVisitor, network[i]); } diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 0da69fc02b..2cc3e3fb98 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -96,6 +96,10 @@ set(SOURCES weight_norm_impl.hpp hardshrink.hpp hardshrink_impl.hpp + celu.hpp + celu_impl.hpp + softshrink.hpp + softshrink_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/ann/layer/add.hpp b/src/mlpack/methods/ann/layer/add.hpp index 47de360eed..12a121414b 100644 --- a/src/mlpack/methods/ann/layer/add.hpp +++ b/src/mlpack/methods/ann/layer/add.hpp @@ -49,7 +49,7 @@ class Add * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -61,9 +61,9 @@ class Add * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - const arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activation. @@ -73,9 +73,9 @@ class Add * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/add_impl.hpp b/src/mlpack/methods/ann/layer/add_impl.hpp index dda53d94dd..10e7ca3a1c 100644 --- a/src/mlpack/methods/ann/layer/add_impl.hpp +++ b/src/mlpack/methods/ann/layer/add_impl.hpp @@ -29,7 +29,7 @@ Add::Add(const size_t outSize) : template template void Add::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { output = input; output.each_col() += weights; @@ -38,9 +38,9 @@ void Add::Forward( template template void Add::Backward( - const arma::Mat&& /* input */, - const arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy; } @@ -48,9 +48,9 @@ void Add::Backward( template template void Add::Gradient( - const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient) { gradient = error; } diff --git a/src/mlpack/methods/ann/layer/add_merge.hpp b/src/mlpack/methods/ann/layer/add_merge.hpp index 28ffb2359f..7b47988a79 100644 --- a/src/mlpack/methods/ann/layer/add_merge.hpp +++ b/src/mlpack/methods/ann/layer/add_merge.hpp @@ -50,6 +50,15 @@ class AddMerge */ AddMerge(const bool model = false, const bool run = true); + /** + * Create the AddMerge object using the specified parameters. + * + * @param model Expose all the network modules. + * @param run Call the Forward/Backward method before the output is merged. + * @param ownsLayers Delete the layers when this is deallocated. + */ + AddMerge(const bool model, const bool run, const bool ownsLayers); + //! Destructor to release allocated memory. ~AddMerge(); @@ -61,7 +70,7 @@ class AddMerge * @param output Resulting output activation. */ template - void Forward(InputType&& /* input */, OutputType&& output); + void Forward(const InputType& /* input */, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -73,9 +82,9 @@ class AddMerge * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * This is the overload of Backward() that runs only a specific layer with @@ -87,9 +96,9 @@ class AddMerge * @param The index of the layer to run. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g, + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g, const size_t index); /* @@ -100,9 +109,9 @@ class AddMerge * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); /* * This is the overload of Gradient() that runs a specific layer with the @@ -114,9 +123,9 @@ class AddMerge * @param The index of the layer to run. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient, + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient, const size_t index); /* @@ -184,8 +193,9 @@ class AddMerge //! before merging the output. bool run; - //! We need this to know whether we should delete the layer in the destructor. - bool ownsLayer; + //! We need this to know whether we should delete the internally-held layers + //! in the destructor. + bool ownsLayers; //! Locally-stored network modules. std::vector > network; @@ -221,6 +231,24 @@ class AddMerge } // namespace ann } // namespace mlpack +//! Set the serialization version of the AddMerge class. +namespace boost { +namespace serialization { + +template< + typename InputDataType, + typename OutputDataType, + typename... CustomLayers +> +struct version> +{ + BOOST_STATIC_CONSTANT(int, value = 1); +}; + +} // namespace serialization +} // namespace boost + // Include implementation. #include "add_merge_impl.hpp" diff --git a/src/mlpack/methods/ann/layer/add_merge_impl.hpp b/src/mlpack/methods/ann/layer/add_merge_impl.hpp index 9fd717a2d1..d3cd789d6a 100644 --- a/src/mlpack/methods/ann/layer/add_merge_impl.hpp +++ b/src/mlpack/methods/ann/layer/add_merge_impl.hpp @@ -27,7 +27,16 @@ template AddMerge::AddMerge( const bool model, const bool run) : - model(model), run(run), ownsLayer(!model) + model(model), run(run), ownsLayers(!model) +{ + // Nothing to do here. +} + +template +AddMerge::AddMerge( + const bool model, const bool run, const bool ownsLayers) : + model(model), run(run), ownsLayers(ownsLayers) { // Nothing to do here. } @@ -36,7 +45,7 @@ template AddMerge::~AddMerge() { - if (ownsLayer) + if (!model && ownsLayers) { std::for_each(network.begin(), network.end(), boost::apply_visitor(deleteVisitor)); @@ -47,14 +56,14 @@ template template void AddMerge::Forward( - InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { if (run) { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, network[i]))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network[i])), network[i]); } } @@ -70,15 +79,17 @@ template template void AddMerge::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { if (run) { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[i])), std::move(gy), std::move( - boost::apply_visitor(deltaVisitor, network[i]))), network[i]); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i]), gy, + boost::apply_visitor(deltaVisitor, network[i])), network[i]); } g = boost::apply_visitor(deltaVisitor, network[0]); @@ -95,12 +106,14 @@ template template void AddMerge::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g, + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g, const size_t index) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[index])), std::move(gy), std::move( - boost::apply_visitor(deltaVisitor, network[index]))), network[index]); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[index]), gy, + boost::apply_visitor(deltaVisitor, network[index])), network[index]); g = boost::apply_visitor(deltaVisitor, network[index]); } @@ -108,16 +121,15 @@ template template void AddMerge::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */ ) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */ ) { if (run) { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), - network[i]); + boost::apply_visitor(GradientVisitor(input, error), network[i]); } } } @@ -126,20 +138,19 @@ template template void AddMerge::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */, + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */, const size_t index) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), - network[index]); + boost::apply_visitor(GradientVisitor(input, error), network[index]); } template template void AddMerge::serialize( - Archive& ar, const unsigned int /* version */) + Archive& ar, const unsigned int version) { // Be sure to clear other layers before loading. if (Archive::is_loading::value) @@ -148,7 +159,11 @@ void AddMerge::serialize( ar & BOOST_SERIALIZATION_NVP(network); ar & BOOST_SERIALIZATION_NVP(model); ar & BOOST_SERIALIZATION_NVP(run); - ar & BOOST_SERIALIZATION_NVP(ownsLayer); + + if (version >= 1) + ar & BOOST_SERIALIZATION_NVP(ownsLayers); + else if (Archive::is_loading::value) + ownsLayers = !model; } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/alpha_dropout.hpp b/src/mlpack/methods/ann/layer/alpha_dropout.hpp index 4edbf3031c..2713bf3c78 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout.hpp @@ -35,7 +35,8 @@ namespace ann /** Artificial Neural Network. */ { * Andreas Mayr}, * title = {Self-Normalizing Neural Networks}, * journal = {Advances in Neural Information Processing Systems}, - * year = {2017} + * year = {2017}, + * url = {https://arxiv.org/abs/1706.02515} * } * @endcode * @@ -65,7 +66,7 @@ class AlphaDropout * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the alpha_dropout layer. @@ -75,9 +76,9 @@ class AlphaDropout * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp b/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp index 78a7790536..fdeac03952 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp @@ -36,7 +36,7 @@ AlphaDropout::AlphaDropout( template template void AlphaDropout::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { // The dropout mask will not be multiplied in the deterministic mode // (during testing). @@ -58,7 +58,7 @@ void AlphaDropout::Forward( template template void AlphaDropout::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = gy % mask * a; } diff --git a/src/mlpack/methods/ann/layer/atrous_convolution.hpp b/src/mlpack/methods/ann/layer/atrous_convolution.hpp index 36d4f9618b..617ba4125f 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution.hpp @@ -137,7 +137,7 @@ class AtrousConvolution * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -149,9 +149,9 @@ class AtrousConvolution * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -161,9 +161,9 @@ class AtrousConvolution * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. const OutputDataType& Parameters() const { return weights; } @@ -365,9 +365,6 @@ class AtrousConvolution //! Locally-stored transformed output parameter. arma::cube outputTemp; - //! Locally-stored transformed input parameter. - arma::cube inputTemp; - //! Locally-stored transformed padded input parameter. arma::cube inputPaddedTemp; diff --git a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp index 32362f71a2..02bdfbfe28 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp @@ -184,10 +184,10 @@ void AtrousConvolution< GradientConvolutionRule, InputDataType, OutputDataType ->::Forward(const arma::Mat&& input, arma::Mat&& output) +>::Forward(const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; - inputTemp = arma::cube(const_cast&&>(input).memptr(), + arma::cube inputTemp(const_cast&>(input).memptr(), inputWidth, inputHeight, inSize * batchSize, false, false); if (padding.PadWLeft() != 0 || padding.PadWRight() != 0 || @@ -200,8 +200,7 @@ void AtrousConvolution< for (size_t i = 0; i < inputTemp.n_slices; ++i) { - padding.Forward(std::move(inputTemp.slice(i)), - std::move(inputPaddedTemp.slice(i))); + padding.Forward(inputTemp.slice(i), inputPaddedTemp.slice(i)); } } @@ -267,14 +266,14 @@ void AtrousConvolution< InputDataType, OutputDataType >::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { - arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, - outSize * batchSize, false, false); + arma::cube mappedError(((arma::Mat&) gy).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); - gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, - inputTemp.n_cols, inputTemp.n_slices, false, false); + g.set_size(inputWidth * inputHeight * inSize, batchSize); + gTemp = arma::Cube(g.memptr(), inputWidth, inputHeight, + inSize * batchSize, false, false); gTemp.zeros(); for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < @@ -326,12 +325,14 @@ void AtrousConvolution< InputDataType, OutputDataType >::Gradient( - const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - arma::cube mappedError(error.memptr(), outputWidth, outputHeight, - outSize * batchSize, false, false); + arma::cube mappedError(((arma::Mat&) error).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); + arma::cube inputTemp(const_cast&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); gradient.set_size(weights.n_elem, 1); gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index c08185488b..6559845d43 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -69,7 +69,7 @@ class BaseLayer * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output) + void Forward(const InputType& input, OutputType& output) { ActivationFunction::Fn(input, output); } @@ -84,9 +84,9 @@ class BaseLayer * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g) + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { arma::Mat derivative; ActivationFunction::Deriv(input, derivative); diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index e22facb9fe..80f220f15a 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -81,7 +81,7 @@ class BatchNorm * @param output Resulting output activations. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Backward pass through the layer. @@ -91,9 +91,9 @@ class BatchNorm * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activations. @@ -103,9 +103,9 @@ class BatchNorm * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 3c1922dce1..81c75d1997 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -63,7 +63,7 @@ void BatchNorm::Reset() template template void BatchNorm::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { // Mean and variance over the entire training set will be used to compute // the forward pass when deterministic is set to true. @@ -106,7 +106,7 @@ void BatchNorm::Forward( template template void BatchNorm::Backward( - const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) { const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); @@ -130,9 +130,9 @@ void BatchNorm::Backward( template template void BatchNorm::Gradient( - const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient) { gradient.set_size(size + size, 1); diff --git a/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp index 9ea1e20bbc..c80f0f4b57 100644 --- a/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp @@ -65,7 +65,7 @@ class BilinearInterpolation * @param output The resulting interpolated output matrix. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -79,9 +79,9 @@ class BilinearInterpolation * @param output The resulting down-sampled output. */ template - void Backward(const arma::Mat&& /*input*/, - arma::Mat&& gradient, - arma::Mat&& output); + void Backward(const arma::Mat& /*input*/, + const arma::Mat& gradient, + arma::Mat& output); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp index bd22e2684a..e2e1c005e7 100644 --- a/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp @@ -54,7 +54,7 @@ BilinearInterpolation( template template void BilinearInterpolation::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; if (output.is_empty()) @@ -68,7 +68,7 @@ void BilinearInterpolation::Forward( assert(inRowSize >= 2); assert(inColSize >= 2); - arma::cube inputAsCube(const_cast&&>(input).memptr(), + arma::cube inputAsCube(const_cast&>(input).memptr(), inRowSize, inColSize, depth * batchSize, false, false); arma::cube outputAsCube(output.memptr(), outRowSize, outColSize, depth * batchSize, false, true); @@ -114,9 +114,9 @@ void BilinearInterpolation::Forward( template template void BilinearInterpolation::Backward( - const arma::Mat&& /*input*/, - arma::Mat&& gradient, - arma::Mat&& output) + const arma::Mat& /*input*/, + const arma::Mat& gradient, + arma::Mat& output) { if (output.is_empty()) output.set_size(inRowSize * inColSize * depth, batchSize); @@ -129,8 +129,8 @@ void BilinearInterpolation::Backward( assert(outRowSize >= 2); assert(outColSize >= 2); - arma::cube gradientAsCube(gradient.memptr(), outRowSize, outColSize, - depth * batchSize, false, false); + arma::cube gradientAsCube(((arma::Mat&) gradient).memptr(), outRowSize, + outColSize, depth * batchSize, false, false); arma::cube outputAsCube(output.memptr(), inRowSize, inColSize, depth * batchSize, false, true); diff --git a/src/mlpack/methods/ann/layer/c_relu.hpp b/src/mlpack/methods/ann/layer/c_relu.hpp index 5108646116..f41a3f77cf 100644 --- a/src/mlpack/methods/ann/layer/c_relu.hpp +++ b/src/mlpack/methods/ann/layer/c_relu.hpp @@ -33,7 +33,8 @@ namespace ann /** Artificial Neural Network. */ { * title = {Understanding and Improving Convolutional Neural Networks * via Concatenated Rectified Linear Units}, * author = {LWenling Shang, Kihyuk Sohn, Diogo Almeida, Honglak Lee}, - * year = {2016} + * year = {2016}, + * url = {https://arxiv.org/abs/1603.05201} * } * @endcode * @@ -57,13 +58,13 @@ class CReLU /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. - * Works only for 2D Tenosrs. + * Works only for 2D Tensors. * * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -75,7 +76,7 @@ class CReLU * @param g The calculated gradient. */ template - void Backward(const DataType&& input, DataType&& gy, DataType&& g); + void Backward(const DataType& input, const DataType& gy, DataType& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/c_relu_impl.hpp b/src/mlpack/methods/ann/layer/c_relu_impl.hpp index e839526976..1bac0a179d 100644 --- a/src/mlpack/methods/ann/layer/c_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/c_relu_impl.hpp @@ -27,7 +27,7 @@ CReLU::CReLU() template template void CReLU::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { output = arma::join_cols(arma::max(input, 0.0 * input), arma::max( (-1 * input), 0.0 * input)); @@ -36,7 +36,7 @@ void CReLU::Forward( template template void CReLU::Backward( - const DataType&& input, DataType&& gy, DataType&& g) + const DataType& input, const DataType& gy, DataType& g) { DataType temp; temp = gy % (input >= 0.0); diff --git a/src/mlpack/methods/ann/layer/celu.hpp b/src/mlpack/methods/ann/layer/celu.hpp new file mode 100644 index 0000000000..a64d128489 --- /dev/null +++ b/src/mlpack/methods/ann/layer/celu.hpp @@ -0,0 +1,143 @@ +/** + * @file celu.hpp + * @author Gaurav Singh + * + * Definition of the CELU activation function as described by Jonathan T. Barron. + * + * For more information, read the following paper. + * + * @code + * @article{ + * author = {Jonathan T. Barron}, + * title = {Continuously Differentiable Exponential Linear Units}, + * year = {2017}, + * url = {https://arxiv.org/pdf/1704.07483} + * } + * @endcode + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_CELU_HPP +#define MLPACK_METHODS_ANN_LAYER_CELU_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The CELU activation function, defined by + * + * @f{eqnarray*}{ + * f(x) &=& \left\{ + * \begin{array}{lr} + * x & : x \ge 0 \\ + * \alpha(e^(\frac{x}{\alpha}) - 1) & : x < 0 + * \end{array} + * \right. \\ + * f'(x) &=& \left\{ + * \begin{array}{lr} + * 1 & : x \ge 0 \\ + * (\frac{f(x)}{\alpha}) + 1 & : x < 0 + * \end{array} + * \right. + * @f} + * + * In the deterministic mode, there is no computation of the derivative. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class CELU +{ + public: + /** + * Create the CELU object using the specified parameter. The non zero + * gradient for negative inputs can be adjusted by specifying the CELU + * hyperparameter alpha (alpha > 0). + * + * @param alpha Scale parameter for the negative factor (default = 1.0). + */ + CELU(const double alpha = 1.0); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const InputType& input, OutputType& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards through f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation f(x). + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const DataType& input, const DataType& gy, DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the non zero gradient. + double const& Alpha() const { return alpha; } + //! Modify the non zero gradient. + double& Alpha() { return alpha; } + + //! Get the value of deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of deterministic parameter. + bool& Deterministic() { return deterministic; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally stored first derivative of the activation function. + arma::mat derivative; + + //! CELU Hyperparameter (alpha > 0). + double alpha; + + //! If true the derivative computation is disabled, see notes above. + bool deterministic; +}; // class CELU + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "celu_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/celu_impl.hpp b/src/mlpack/methods/ann/layer/celu_impl.hpp new file mode 100644 index 0000000000..bf92e5da1a --- /dev/null +++ b/src/mlpack/methods/ann/layer/celu_impl.hpp @@ -0,0 +1,76 @@ +/** + * @file celu_impl.hpp + * @author Gaurav Singh + * + * Implementation of the CELU activation function as described by Jonathan T. Barron. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_CELU_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_CELU_IMPL_HPP + +// In case it hasn't yet been included. +#include "celu.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +CELU::CELU(const double alpha) : + alpha(alpha), + deterministic(false) +{ + if (alpha == 0) + { + Log::Fatal << "The value of alpha cannot be equal to 0, " + << "terminating the program." << std::endl; + } +} + +template +template +void CELU::Forward( + const InputType& input, OutputType& output) +{ + output = arma::ones(arma::size(input)); + for (size_t i = 0; i < input.n_elem; i++) + { + output(i) = (input(i) >= 0) ? input(i) : alpha * + (std::exp(input(i) / alpha) - 1); + } + + if (!deterministic) + { + derivative.set_size(arma::size(input)); + for (size_t i = 0; i < input.n_elem; i++) + { + derivative(i) = (input(i) >= 0) ? 1 : + (output(i) / alpha) + 1; + } + } +} + +template +template +void CELU::Backward( + const DataType& /* input */, const DataType& gy, DataType& g) +{ + g = gy % derivative; +} + +template +template +void CELU::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(alpha); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/concat.hpp b/src/mlpack/methods/ann/layer/concat.hpp index d180681f94..51c2bf2971 100644 --- a/src/mlpack/methods/ann/layer/concat.hpp +++ b/src/mlpack/methods/ann/layer/concat.hpp @@ -3,7 +3,7 @@ * @author Marcus Edel * @author Mehul Kumar Nirala * - * Definition of the Concat class, which acts as a concatenation contain. + * Definition of the Concat class, which acts as a concatenation container. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -80,7 +80,7 @@ class Concat * @param output Resulting output activation. */ template - void Forward(arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, using 3rd-order tensors as @@ -92,9 +92,9 @@ class Concat * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * This is the overload of Backward() that runs only a specific layer with @@ -106,9 +106,9 @@ class Concat * @param The index of the layer to run. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g, + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g, const size_t index); /* @@ -119,9 +119,9 @@ class Concat * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& /* gradient */); + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& /* gradient */); /* * This is the overload of Gradient() that runs a specific layer with the @@ -133,9 +133,9 @@ class Concat * @param The index of the layer to run. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient, + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient, const size_t index); /* diff --git a/src/mlpack/methods/ann/layer/concat_impl.hpp b/src/mlpack/methods/ann/layer/concat_impl.hpp index 7c2e9a82fb..0aeee24fa2 100644 --- a/src/mlpack/methods/ann/layer/concat_impl.hpp +++ b/src/mlpack/methods/ann/layer/concat_impl.hpp @@ -90,23 +90,26 @@ template Concat::~Concat() { - // Clear memory. - std::for_each(network.begin(), network.end(), - boost::apply_visitor(deleteVisitor)); + if (!model) + { + // Clear memory. + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deleteVisitor)); + } } template template void Concat::Forward( - arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { if (run) { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, network[i]))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network[i])), network[i]); } } @@ -134,13 +137,14 @@ template template void Concat::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { size_t rowCount = 0; if (run) { arma::Mat delta; - gy.reshape(gy.n_rows / channels, gy.n_cols * channels); + arma::Mat gyTmp(((arma::Mat&) gy).memptr(), gy.n_rows / channels, + gy.n_cols * channels, false, false); for (size_t i = 0; i < network.size(); ++i) { // Use rows from the error corresponding to the output from each layer. @@ -148,13 +152,13 @@ void Concat::Backward( outputParameterVisitor, network[i]).n_rows; // Extract from gy the parameters for the i-th network. - delta = gy.rows(rowCount / channels, (rowCount + rows) / channels - 1); + delta = gyTmp.rows(rowCount / channels, (rowCount + rows) / channels - 1); delta.reshape(delta.n_rows * channels, delta.n_cols / channels); - boost::apply_visitor(BackwardVisitor(std::move( + boost::apply_visitor(BackwardVisitor( boost::apply_visitor(outputParameterVisitor, - network[i])), std::move(delta), std::move( - boost::apply_visitor(deltaVisitor, network[i]))), network[i]); + network[i]), delta, + boost::apply_visitor(deltaVisitor, network[i])), network[i]); rowCount += rows; } @@ -163,7 +167,6 @@ void Concat::Backward( { g += boost::apply_visitor(deltaVisitor, network[i]); } - gy.reshape(gy.n_rows * channels, gy.n_cols / channels); } else { @@ -175,7 +178,9 @@ template template void Concat::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g, + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g, const size_t index) { size_t rowCount = 0, rows = 0; @@ -188,18 +193,16 @@ void Concat::Backward( rows = boost::apply_visitor(outputParameterVisitor, network[index]).n_rows; // Reshape gy to extract the i-th layer gy. - gy.reshape(gy.n_rows / channels, gy.n_cols * channels); + arma::Mat gyTmp(((arma::Mat&) gy).memptr(), gy.n_rows / channels, + gy.n_cols * channels, false, false); - arma::Mat delta = gy.rows(rowCount / channels, (rowCount + rows) / + arma::Mat delta = gyTmp.rows(rowCount / channels, (rowCount + rows) / channels - 1); delta.reshape(delta.n_rows * channels, delta.n_cols / channels); - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[index])), std::move(delta), std::move( - boost::apply_visitor(deltaVisitor, network[index]))), network[index]); - - // Reshape gy to its original shape. - gy.reshape(gy.n_rows * channels, gy.n_cols / channels); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[index]), delta, + boost::apply_visitor(deltaVisitor, network[index])), network[index]); g = boost::apply_visitor(deltaVisitor, network[index]); } @@ -208,32 +211,29 @@ template template void Concat::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */) { if (run) { size_t rowCount = 0; // Reshape error to extract the i-th layer error. - error.reshape(error.n_rows / channels, error.n_cols * channels); + arma::Mat errorTmp(((arma::Mat&) error).memptr(), + error.n_rows / channels, error.n_cols * channels, false, false); for (size_t i = 0; i < network.size(); ++i) { size_t rows = boost::apply_visitor( outputParameterVisitor, network[i]).n_rows; // Extract from error the parameters for the i-th network. - arma::Mat err = error.rows(rowCount / channels, (rowCount + rows) / + arma::Mat err = errorTmp.rows(rowCount / channels, (rowCount + rows) / channels - 1); err.reshape(err.n_rows * channels, err.n_cols / channels); - boost::apply_visitor(GradientVisitor(std::move(input), - std::move(err)), network[i]); + boost::apply_visitor(GradientVisitor(input, err), network[i]); rowCount += rows; } - - // Reshape error to its original shape. - error.reshape(error.n_rows * channels, error.n_cols / channels); } } @@ -241,9 +241,9 @@ template template void Concat::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */, + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */, const size_t index) { size_t rowCount = 0; @@ -255,15 +255,13 @@ void Concat::Gradient( size_t rows = boost::apply_visitor( outputParameterVisitor, network[index]).n_rows; - error.reshape(error.n_rows / channels, error.n_cols * channels); - arma::Mat err = error.rows(rowCount / channels, (rowCount + rows) / + arma::Mat errorTmp(((arma::Mat&) error).memptr(), + error.n_rows / channels, error.n_cols * channels, false, false); + arma::Mat err = errorTmp.rows(rowCount / channels, (rowCount + rows) / channels - 1); err.reshape(err.n_rows * channels, err.n_cols / channels); - boost::apply_visitor(GradientVisitor(std::move(input), - std::move(err)), network[index]); - - error.reshape(error.n_rows * channels, error.n_cols / channels); + boost::apply_visitor(GradientVisitor(input, err), network[index]); } template - double Forward(const arma::Mat&& input, arma::Mat&& target); + double Forward(const arma::Mat& input, arma::Mat& target); + /** * Ordinary feed backward pass of a neural network. The negative log * likelihood layer expectes that the input contains log-probabilities for @@ -68,9 +69,9 @@ class ConcatPerformance * @param output The calculated error. */ template - void Backward(const arma::Mat&& input, - const arma::Mat&& target, - arma::Mat&& output); + void Backward(const arma::Mat& input, + const arma::Mat& target, + arma::Mat& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/concat_performance_impl.hpp b/src/mlpack/methods/ann/layer/concat_performance_impl.hpp index b4c2d6b9e1..96894bc404 100644 --- a/src/mlpack/methods/ann/layer/concat_performance_impl.hpp +++ b/src/mlpack/methods/ann/layer/concat_performance_impl.hpp @@ -44,7 +44,7 @@ double ConcatPerformance< OutputLayerType, InputDataType, OutputDataType ->::Forward(const arma::Mat&& input, arma::Mat&& target) +>::Forward(const arma::Mat& input, arma::Mat& target) { const size_t elements = input.n_elem / inSize; @@ -52,7 +52,7 @@ double ConcatPerformance< for (size_t i = 0; i < input.n_elem; i+= elements) { arma::mat subInput = input.submat(i, 0, i + elements - 1, 0); - output += outputLayer.Forward(std::move(subInput), std::move(target)); + output += outputLayer.Forward(subInput, target); } return output; @@ -69,17 +69,16 @@ void ConcatPerformance< InputDataType, OutputDataType >::Backward( - const arma::Mat&& input, - const arma::Mat&& target, - arma::Mat&& output) + const arma::Mat& input, + const arma::Mat& target, + arma::Mat& output) { const size_t elements = input.n_elem / inSize; arma::mat subInput = input.submat(0, 0, elements - 1, 0); arma::mat subOutput; - outputLayer.Backward(std::move(subInput), std::move(target), - std::move(subOutput)); + outputLayer.Backward(subInput, target, subOutput); output = arma::zeros(subOutput.n_elem, inSize); output.col(0) = subOutput; @@ -87,8 +86,7 @@ void ConcatPerformance< for (size_t i = elements, j = 0; i < input.n_elem; i+= elements, j++) { subInput = input.submat(i, 0, i + elements - 1, 0); - outputLayer.Backward(std::move(subInput), std::move(target), - std::move(subOutput)); + outputLayer.Backward(subInput, target, subOutput); output.col(j) = subOutput; } @@ -104,9 +102,9 @@ void ConcatPerformance< OutputLayerType, InputDataType, OutputDataType ->::serialize(Archive& /* ar */, const unsigned int /* version */) +>::serialize(Archive& ar, const unsigned int /* version */) { - // Nothing to do here. + ar & BOOST_SERIALIZATION_NVP(inSize); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/concatenate.hpp b/src/mlpack/methods/ann/layer/concatenate.hpp index 77fd985cf4..232da963d6 100644 --- a/src/mlpack/methods/ann/layer/concatenate.hpp +++ b/src/mlpack/methods/ann/layer/concatenate.hpp @@ -49,7 +49,7 @@ class Concatenate * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -61,9 +61,9 @@ class Concatenate * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - const arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/concatenate_impl.hpp b/src/mlpack/methods/ann/layer/concatenate_impl.hpp index c1e3139f67..743c9b8f2f 100644 --- a/src/mlpack/methods/ann/layer/concatenate_impl.hpp +++ b/src/mlpack/methods/ann/layer/concatenate_impl.hpp @@ -28,7 +28,7 @@ Concatenate::Concatenate() template template void Concatenate::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { if (concat.is_empty()) Log::Warn << "The concat matrix has not been provided." << std::endl; @@ -46,9 +46,9 @@ void Concatenate::Forward( template template void Concatenate::Backward( - const arma::Mat&& /* input */, - const arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy.submat(0, 0, inRows - 1, concat.n_cols - 1); } diff --git a/src/mlpack/methods/ann/layer/constant.hpp b/src/mlpack/methods/ann/layer/constant.hpp index 62ed80faa9..c3f84f8ab0 100644 --- a/src/mlpack/methods/ann/layer/constant.hpp +++ b/src/mlpack/methods/ann/layer/constant.hpp @@ -51,7 +51,7 @@ class Constant * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network. The backward pass of the @@ -62,9 +62,9 @@ class Constant * @param g The calculated gradient. */ template - void Backward(const DataType&& /* input */, - DataType&& /* gy */, - DataType&& g); + void Backward(const DataType& /* input */, + const DataType& /* gy */, + DataType& g); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/constant_impl.hpp b/src/mlpack/methods/ann/layer/constant_impl.hpp index 429cd44805..8a842c7ff4 100644 --- a/src/mlpack/methods/ann/layer/constant_impl.hpp +++ b/src/mlpack/methods/ann/layer/constant_impl.hpp @@ -33,7 +33,7 @@ Constant::Constant( template template void Constant::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { if (inSize == 0) { @@ -46,7 +46,7 @@ void Constant::Forward( template template void Constant::Backward( - const DataType&& /* input */, DataType&& /* gy */, DataType&& g) + const DataType& /* input */, const DataType& /* gy */, DataType& g) { g = arma::zeros(inSize, 1); } diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index 11352ec371..bad950d2d7 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -123,7 +123,7 @@ class Convolution * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -135,9 +135,9 @@ class Convolution * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -147,9 +147,9 @@ class Convolution * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. const OutputDataType& Parameters() const { return weights; } @@ -362,9 +362,6 @@ class Convolution //! Locally-stored transformed output parameter. arma::cube outputTemp; - //! Locally-stored transformed input parameter. - arma::cube inputTemp; - //! Locally-stored transformed padded input parameter. arma::cube inputPaddedTemp; diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 5f44836a3c..d9c5886a10 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -71,7 +71,8 @@ Convolution< std::tuple(padW, padW), std::tuple(padH, padH), inputWidth, - inputHeight) + inputHeight, + paddingType) { // Nothing to do here. } @@ -174,10 +175,10 @@ void Convolution< GradientConvolutionRule, InputDataType, OutputDataType ->::Forward(const arma::Mat&& input, arma::Mat&& output) +>::Forward(const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; - inputTemp = arma::cube(const_cast&&>(input).memptr(), + arma::cube inputTemp(const_cast&>(input).memptr(), inputWidth, inputHeight, inSize * batchSize, false, false); if (padWLeft != 0 || padWRight != 0 || padHTop != 0 || padHBottom != 0) @@ -187,8 +188,7 @@ void Convolution< for (size_t i = 0; i < inputTemp.n_slices; ++i) { - padding.Forward(std::move(inputTemp.slice(i)), - std::move(inputPaddedTemp.slice(i))); + padding.Forward(inputTemp.slice(i), inputPaddedTemp.slice(i)); } } @@ -253,14 +253,14 @@ void Convolution< InputDataType, OutputDataType >::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { - arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, - outSize * batchSize, false, false); + arma::cube mappedError(((arma::Mat&) gy).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); - gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, - inputTemp.n_cols, inputTemp.n_slices, false, false); + g.set_size(inputWidth * inputHeight * inSize, batchSize); + gTemp = arma::Cube(g.memptr(), inputWidth, inputHeight, + inSize * batchSize, false, false); gTemp.zeros(); for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < @@ -308,12 +308,14 @@ void Convolution< InputDataType, OutputDataType >::Gradient( - const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - arma::cube mappedError(error.memptr(), outputWidth, + arma::cube mappedError(((arma::Mat&) error).memptr(), outputWidth, outputHeight, outSize * batchSize, false, false); + arma::cube inputTemp(((arma::Mat&) input).memptr(), inputWidth, + inputHeight, inSize * batchSize, false, false); gradient.set_size(weights.n_elem, 1); gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index 1ed6f2ce0c..c1e5c51a68 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -46,7 +46,8 @@ namespace ann /** Artificial Neural Network. */ { * Learning(ICML - 13)}, * author = {Li Wan and Matthew Zeiler and Sixin Zhang and Yann L. Cun and * Rob Fergus}, - * year = {2013} + * year = {2013}, + * url = {http://proceedings.mlr.press/v28/wan13.pdf} * } * @endcode * @@ -77,16 +78,14 @@ class DropConnect const size_t outSize, const double ratio = 0.5); - ~DropConnect(); - /** - * Ordinary feed forward pass of the DropConnect layer. - * - * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. - */ + * Ordinary feed forward pass of the DropConnect layer. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ template - void Forward(arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the DropConnect layer. @@ -96,9 +95,9 @@ class DropConnect * @param g The calculated gradient. */ template - void Backward(arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activation. @@ -108,9 +107,9 @@ class DropConnect * @param g The calculated gradient. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */); //! Get the model modules. std::vector >& Model() { return network; } diff --git a/src/mlpack/methods/ann/layer/dropconnect_impl.hpp b/src/mlpack/methods/ann/layer/dropconnect_impl.hpp index 7a3713ac38..194b4d7fda 100644 --- a/src/mlpack/methods/ann/layer/dropconnect_impl.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect_impl.hpp @@ -48,40 +48,32 @@ DropConnect::DropConnect( network.push_back(baseLayer); } -template -DropConnect::~DropConnect() -{ - boost::apply_visitor(DeleteVisitor(), baseLayer); -} - template template void DropConnect::Forward( - arma::Mat&& input, - arma::Mat&& output) + const arma::Mat& input, + arma::Mat& output) { // The DropConnect mask will not be multiplied in the deterministic mode // (during testing). if (deterministic) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move(output)), - baseLayer); + boost::apply_visitor(ForwardVisitor(input, output), baseLayer); } else { // Save weights for denoising. - boost::apply_visitor(ParametersVisitor(std::move(denoise)), baseLayer); + boost::apply_visitor(ParametersVisitor(denoise), baseLayer); // Scale with input / (1 - ratio) and set values to zero with // probability ratio. mask = arma::randu >(denoise.n_rows, denoise.n_cols); mask.transform([&](double val) { return (val > ratio); }); - boost::apply_visitor(ParametersSetVisitor(std::move(denoise % mask)), - baseLayer); + arma::mat tmp = denoise % mask; + boost::apply_visitor(ParametersSetVisitor(tmp), baseLayer); - boost::apply_visitor(ForwardVisitor(std::move(input), std::move(output)), - baseLayer); + boost::apply_visitor(ForwardVisitor(input, output), baseLayer); output = output * scale; } @@ -90,26 +82,25 @@ void DropConnect::Forward( template template void DropConnect::Backward( - arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { - boost::apply_visitor(BackwardVisitor(std::move(input), std::move(gy), - std::move(g)), baseLayer); + boost::apply_visitor(BackwardVisitor(input, gy, g), baseLayer); } template template void DropConnect::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), + boost::apply_visitor(GradientVisitor(input, error), baseLayer); // Denoise the weights. - boost::apply_visitor(ParametersSetVisitor(std::move(denoise)), baseLayer); + boost::apply_visitor(ParametersSetVisitor(denoise), baseLayer); } template diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp index 696d92869c..ade2bfa7ab 100644 --- a/src/mlpack/methods/ann/layer/dropout.hpp +++ b/src/mlpack/methods/ann/layer/dropout.hpp @@ -39,6 +39,7 @@ namespace ann /** Artificial Neural Network. */ { * journal = {CoRR}, * volume = {abs/1207.0580}, * year = {2012}, + * url = {https://arxiv.org/abs/1207.0580} * } * @endcode * @@ -66,7 +67,7 @@ class Dropout * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the dropout layer. @@ -76,9 +77,9 @@ class Dropout * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/dropout_impl.hpp b/src/mlpack/methods/ann/layer/dropout_impl.hpp index ccac9588f8..750b11958c 100644 --- a/src/mlpack/methods/ann/layer/dropout_impl.hpp +++ b/src/mlpack/methods/ann/layer/dropout_impl.hpp @@ -32,8 +32,8 @@ Dropout::Dropout( template template void Dropout::Forward( - const arma::Mat&& input, - arma::Mat&& output) + const arma::Mat& input, + arma::Mat& output) { // The dropout mask will not be multiplied in the deterministic mode // (during testing). @@ -54,9 +54,9 @@ void Dropout::Forward( template template void Dropout::Backward( - const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy % mask * scale; } diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 575b99e4d5..200d873f81 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -3,7 +3,7 @@ * @author Vivek Pal * @author Dakshit Agrawal * - * Definition of the ELU activation function as descibed by Djork-Arne Clevert, + * Definition of the ELU activation function as described by Djork-Arne Clevert, * Thomas Unterthiner and Sepp Hochreiter. * * Definition of the SELU function as introduced by @@ -56,7 +56,8 @@ namespace ann /** Artificial Neural Network. */ { * title = {Fast and Accurate Deep Network Learning by Exponential Linear * Units (ELUs)}, * journal = {CoRR}, - * year = {2015} + * year = {2015}, + * url = {https://arxiv.org/abs/1511.07289} * } * @endcode * @@ -86,7 +87,8 @@ namespace ann /** Artificial Neural Network. */ { * Andreas Mayr}, * title = {Self-Normalizing Neural Networks}, * journal = {Advances in Neural Information Processing Systems}, - * year = {2017} + * year = {2017}, + * url = {https://arxiv.org/abs/1706.02515} * } * @endcode * @@ -134,7 +136,7 @@ class ELU * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -146,7 +148,7 @@ class ELU * @param g The calculated gradient. */ template - void Backward(const DataType&& input, DataType&& gy, DataType&& g); + void Backward(const DataType& input, const DataType& gy, DataType& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } @@ -163,6 +165,11 @@ class ELU //! Modify the non zero gradient. double& Alpha() { return alpha; } + //! Get the value of deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of deterministic parameter. + bool& Deterministic() { return deterministic; } + //! Get the lambda parameter. double const& Lambda() const { return lambda; } diff --git a/src/mlpack/methods/ann/layer/elu_impl.hpp b/src/mlpack/methods/ann/layer/elu_impl.hpp index 3c94871e49..cc2d189995 100644 --- a/src/mlpack/methods/ann/layer/elu_impl.hpp +++ b/src/mlpack/methods/ann/layer/elu_impl.hpp @@ -3,7 +3,7 @@ * @author Vivek Pal * @author Dakshit Agrawal * - * Implementation of the ELU activation function as descibed by Djork-Arne + * Implementation of the ELU activation function as described by Djork-Arne * Clevert, Thomas Unterthiner and Sepp Hochreiter. * * Implementation of the SELU function as introduced by Klambauer et. al. in @@ -49,9 +49,9 @@ ELU::ELU(const double alpha) : template template void ELU::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { - output.set_size(arma::size(input)); + output = arma::ones(arma::size(input)); for (size_t i = 0; i < input.n_elem; i++) { if (input(i) < DBL_MAX) @@ -59,8 +59,6 @@ void ELU::Forward( output(i) = (input(i) > 0) ? lambda * input(i) : lambda * alpha * (std::exp(input(i)) - 1); } - else - output(i) = 1.0; } if (!deterministic) @@ -77,7 +75,7 @@ void ELU::Forward( template template void ELU::Backward( - const DataType&& /* input */, DataType&& gy, DataType&& g) + const DataType& /* input */, const DataType& gy, DataType& g) { g = gy % derivative; } diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index e73906a5ba..4d25487641 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -47,7 +47,8 @@ namespace ann /** Artificial Neural Network. */ { * author = {Hochreiter, Sepp and Schmidhuber, J\"{u}rgen}, * title = {Long Short-term Memory}, * journal = {Neural Comput.}, - * year = {1997} + * year = {1997}, + * url = {https://www.bioinf.jku.at/publications/older/2604.pdf} * } * @endcode * @@ -91,7 +92,7 @@ class FastLSTM * @param output Resulting output activation. */ template - void Forward(InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -103,9 +104,9 @@ class FastLSTM * @param g The calculated gradient. */ template - void Backward(const InputType&& input, - ErrorType&& gy, - GradientType&& g); + void Backward(const InputType& input, + const ErrorType& gy, + GradientType& g); /* * Reset the layer parameter. @@ -128,9 +129,9 @@ class FastLSTM * @param gradient The calculated gradient. */ template - void Gradient(InputType&& input, - ErrorType&& error, - GradientType&& gradient); + void Gradient(const InputType& input, + const ErrorType& error, + GradientType& gradient); //! Get the maximum number of steps to backpropagate through time (BPTT). size_t Rho() const { return rho; } @@ -171,7 +172,7 @@ class FastLSTM * @param sigmoid The matrix to store the sigmoid approximation into. */ template - void FastSigmoid(InputType&& input, OutputType&& sigmoids) + void FastSigmoid(const InputType& input, OutputType& sigmoids) { for (size_t i = 0; i < input.n_elem; ++i) sigmoids(i) = FastSigmoid(input(i)); diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp index 36d92c2a0d..47dfb4bf1e 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -112,7 +112,7 @@ void FastLSTM::ResetCell(const size_t size) template template void FastLSTM::Forward( - InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { // Check if the batch size changed, the number of cols is defines the input // batch size. @@ -128,9 +128,11 @@ void FastLSTM::Forward( forwardStep, forwardStep + batchStep); gate.cols(forwardStep, forwardStep + batchStep).each_col() += input2GateBias; - FastSigmoid(std::move( - gate.submat(0, forwardStep, 3 * outSize - 1, forwardStep + batchStep)), - std::move(gateActivation.cols(forwardStep, forwardStep + batchStep))); + arma::subview sigmoidOut = gateActivation.cols(forwardStep, + forwardStep + batchStep); + FastSigmoid( + gate.submat(0, forwardStep, 3 * outSize - 1, forwardStep + batchStep), + sigmoidOut); stateActivation.cols(forwardStep, forwardStep + batchStep) = arma::tanh( gate.submat(3 * outSize, forwardStep, 4 * outSize - 1, @@ -178,14 +180,20 @@ void FastLSTM::Forward( template template void FastLSTM::Backward( - const InputType&& /* input */, ErrorType&& gy, GradientType&& g) + const InputType& /* input */, const ErrorType& gy, GradientType& g) { + ErrorType gyLocal; if (gradientStepIdx > 0) { - gy += output2GateWeight.t() * prevError; + gyLocal = gy + output2GateWeight.t() * prevError; + } + else + { + gyLocal = ErrorType(((ErrorType&) gy).memptr(), gy.n_rows, gy.n_cols, false, + false); } - cellActivationError = gy % gateActivation.submat(outSize, + cellActivationError = gyLocal % gateActivation.submat(outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep) % (1 - arma::pow(cellActivation.cols(backwardStep - batchStep, backwardStep), 2)); @@ -225,7 +233,7 @@ void FastLSTM::Backward( prevError.submat(outSize, 0, 2 * outSize - 1, batchStep) = cellActivation.cols(backwardStep - batchStep, - backwardStep) % gy % gateActivation.submat( + backwardStep) % gyLocal % gateActivation.submat( outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep) % (1.0 - gateActivation.submat( outSize, backwardStep - batchStep, 2 * outSize - 1, backwardStep)); @@ -244,7 +252,9 @@ void FastLSTM::Backward( template template void FastLSTM::Gradient( - InputType&& input, ErrorType&& /* error */, GradientType&& gradient) + const InputType& input, + const ErrorType& /* error */, + GradientType& gradient) { // Gradient of the input to gate layer. gradient.submat(0, 0, input2GateWeight.n_elem - 1, 0) = diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp index 89e013610f..184ecde71d 100644 --- a/src/mlpack/methods/ann/layer/flexible_relu.hpp +++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp @@ -84,7 +84,7 @@ class FlexibleReLU * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -96,7 +96,7 @@ class FlexibleReLU * @param g The calculated gradient. */ template - void Backward(const DataType&& input, DataType&& gy, DataType&& g); + void Backward(const DataType& input, const DataType& gy, DataType& g); /** * Calculate the gradient using the output delta and the input activation. @@ -106,9 +106,9 @@ class FlexibleReLU * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return alpha; } diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp index 52ab56f498..02af24bb38 100644 --- a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp @@ -41,7 +41,7 @@ void FlexibleReLU::Reset() template template void FlexibleReLU::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { output = arma::clamp(input, 0.0, DBL_MAX) + alpha(0); } @@ -49,7 +49,7 @@ void FlexibleReLU::Forward( template template void FlexibleReLU::Backward( - const DataType&& input, DataType&& gy, DataType&& g) + const DataType& input, const DataType& gy, DataType& g) { //! Compute the first derivative of FlexibleReLU function. g = gy % arma::clamp(arma::sign(input), 0.0, 1.0); @@ -58,9 +58,9 @@ void FlexibleReLU::Backward( template template void FlexibleReLU::Gradient( - const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { if (gradient.n_elem == 0) { diff --git a/src/mlpack/methods/ann/layer/glimpse.hpp b/src/mlpack/methods/ann/layer/glimpse.hpp index 357fbce726..ed1017284d 100644 --- a/src/mlpack/methods/ann/layer/glimpse.hpp +++ b/src/mlpack/methods/ann/layer/glimpse.hpp @@ -15,6 +15,7 @@ * journal = {CoRR}, * volume = {abs/1406.6247}, * year = {2014}, + * url = {https://arxiv.org/abs/1406.6247} * } * @endcode * @@ -113,7 +114,7 @@ class Glimpse * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the glimpse layer. @@ -123,9 +124,9 @@ class Glimpse * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType& OutputParameter() const {return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/glimpse_impl.hpp b/src/mlpack/methods/ann/layer/glimpse_impl.hpp index a867656895..49f6854301 100644 --- a/src/mlpack/methods/ann/layer/glimpse_impl.hpp +++ b/src/mlpack/methods/ann/layer/glimpse_impl.hpp @@ -45,7 +45,7 @@ Glimpse::Glimpse( template template void Glimpse::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { inputTemp = arma::cube(input.colptr(0), inputWidth, inputHeight, inSize); outputTemp = arma::Cube(size, size, depth * inputTemp.n_slices); @@ -129,7 +129,7 @@ void Glimpse::Forward( template template void Glimpse::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { // Generate a cube using the backpropagated error matrix. arma::Cube mappedError = arma::zeros(outputWidth, diff --git a/src/mlpack/methods/ann/layer/gru.hpp b/src/mlpack/methods/ann/layer/gru.hpp index 63fee576be..52dd2730c3 100644 --- a/src/mlpack/methods/ann/layer/gru.hpp +++ b/src/mlpack/methods/ann/layer/gru.hpp @@ -13,7 +13,8 @@ Kyunghyun and Bengio, Yoshua}, * booktitle = {ICML}, * pages = {2067--2075}, - * year = {2015} + * year = {2015}, + * url = {https://arxiv.org/abs/1502.02367} * } * @endcode * @@ -71,11 +72,6 @@ class GRU const size_t outSize, const size_t rho = std::numeric_limits::max()); - /** - * Delete the GRU and the layers it holds. - */ - ~GRU(); - /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. @@ -84,7 +80,7 @@ class GRU * @param output Resulting output activation. */ template - void Forward(arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -96,9 +92,9 @@ class GRU * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -108,9 +104,9 @@ class GRU * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& /* error */, - arma::Mat&& /* gradient */); + void Gradient(const arma::Mat& input, + const arma::Mat& /* error */, + arma::Mat& /* gradient */); /* * Resets the cell to accept a new input. This breaks the BPTT chain starts a diff --git a/src/mlpack/methods/ann/layer/gru_impl.hpp b/src/mlpack/methods/ann/layer/gru_impl.hpp index 86c6b85414..b5eb914468 100644 --- a/src/mlpack/methods/ann/layer/gru_impl.hpp +++ b/src/mlpack/methods/ann/layer/gru_impl.hpp @@ -76,21 +76,10 @@ GRU::GRU( gradIterator = outParameter.end(); } -template -GRU::~GRU() -{ - boost::apply_visitor(deleteVisitor, input2GateModule); - boost::apply_visitor(deleteVisitor, output2GateModule); - boost::apply_visitor(deleteVisitor, outputHidden2GateModule); - boost::apply_visitor(deleteVisitor, inputGateModule); - boost::apply_visitor(deleteVisitor, forgetGateModule); - boost::apply_visitor(deleteVisitor, hiddenStateModule); -} - template template void GRU::Forward( - arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { if (input.n_cols != batchSize) { @@ -114,13 +103,13 @@ void GRU::Forward( } // Process the input linearly(zt, rt, ot). - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, input2GateModule))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, input2GateModule)), input2GateModule); // Process the output(zt, rt) linearly. - boost::apply_visitor(ForwardVisitor(std::move(*prevOutput), std::move( - boost::apply_visitor(outputParameterVisitor, output2GateModule))), + boost::apply_visitor(ForwardVisitor(*prevOutput, + boost::apply_visitor(outputParameterVisitor, output2GateModule)), output2GateModule); // Merge the outputs(zt and rt). @@ -129,22 +118,22 @@ void GRU::Forward( boost::apply_visitor(outputParameterVisitor, output2GateModule)); // Pass the first outSize through inputGate(it). - boost::apply_visitor(ForwardVisitor(std::move(output.submat( - 0, 0, 1 * outSize - 1, batchSize - 1)), std::move(boost::apply_visitor( - outputParameterVisitor, inputGateModule))), inputGateModule); + boost::apply_visitor(ForwardVisitor(output.submat( + 0, 0, 1 * outSize - 1, batchSize - 1), boost::apply_visitor( + outputParameterVisitor, inputGateModule)), inputGateModule); // Pass the second through forgetGate. - boost::apply_visitor(ForwardVisitor(std::move(output.submat( - 1 * outSize, 0, 2 * outSize - 1, batchSize - 1)), std::move( - boost::apply_visitor(outputParameterVisitor, forgetGateModule))), + boost::apply_visitor(ForwardVisitor(output.submat( + 1 * outSize, 0, 2 * outSize - 1, batchSize - 1), + boost::apply_visitor(outputParameterVisitor, forgetGateModule)), forgetGateModule); arma::mat modInput = (boost::apply_visitor(outputParameterVisitor, forgetGateModule) % *prevOutput); // Pass that through the outputHidden2GateModule. - boost::apply_visitor(ForwardVisitor(std::move(modInput), std::move( - boost::apply_visitor(outputParameterVisitor, outputHidden2GateModule))), + boost::apply_visitor(ForwardVisitor(modInput, + boost::apply_visitor(outputParameterVisitor, outputHidden2GateModule)), outputHidden2GateModule); // Merge for ot. @@ -153,8 +142,8 @@ void GRU::Forward( boost::apply_visitor(outputParameterVisitor, outputHidden2GateModule); // Pass it through hiddenGate. - boost::apply_visitor(ForwardVisitor(std::move(outputH), std::move( - boost::apply_visitor(outputParameterVisitor, hiddenStateModule))), + boost::apply_visitor(ForwardVisitor(outputH, + boost::apply_visitor(outputParameterVisitor, hiddenStateModule)), hiddenStateModule); // Update the output (nextOutput): cmul1 + cmul2 @@ -205,7 +194,7 @@ void GRU::Forward( template template void GRU::Backward( - const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) { if (input.n_cols != batchSize) { @@ -228,9 +217,15 @@ void GRU::Backward( gradIterator = outParameter.end(); } + arma::Mat gyLocal; if ((outParameter.size() - backwardStep - 1) % rho != 0 && backwardStep != 0) { - gy += boost::apply_visitor(deltaVisitor, output2GateModule); + gyLocal = gy + boost::apply_visitor(deltaVisitor, output2GateModule); + } + else + { + gyLocal = arma::Mat(((arma::Mat&) gy).memptr(), gy.n_rows, + gy.n_cols, false, false); } if (backIterator == outParameter.end()) @@ -239,31 +234,31 @@ void GRU::Backward( } // Delta zt. - arma::mat dZt = gy % (*backIterator - + arma::mat dZt = gyLocal % (*backIterator - boost::apply_visitor(outputParameterVisitor, hiddenStateModule)); // Delta ot. - arma::mat dOt = gy % (arma::ones(outSize, batchSize) - + arma::mat dOt = gyLocal % (arma::ones(outSize, batchSize) - boost::apply_visitor(outputParameterVisitor, inputGateModule)); // Delta of input gate. - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, inputGateModule)), std::move(dZt), - std::move(boost::apply_visitor(deltaVisitor, inputGateModule))), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, inputGateModule), dZt, + boost::apply_visitor(deltaVisitor, inputGateModule)), inputGateModule); // Delta of hidden gate. - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, hiddenStateModule)), std::move(dOt), - std::move(boost::apply_visitor(deltaVisitor, hiddenStateModule))), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, hiddenStateModule), dOt, + boost::apply_visitor(deltaVisitor, hiddenStateModule)), hiddenStateModule); // Delta of outputHidden2GateModule. - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, outputHidden2GateModule)), - std::move(boost::apply_visitor(deltaVisitor, hiddenStateModule)), - std::move(boost::apply_visitor(deltaVisitor, outputHidden2GateModule))), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, outputHidden2GateModule), + boost::apply_visitor(deltaVisitor, hiddenStateModule), + boost::apply_visitor(deltaVisitor, outputHidden2GateModule)), outputHidden2GateModule); // Delta rt. @@ -271,9 +266,9 @@ void GRU::Backward( *backIterator; // Delta of forget gate. - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, forgetGateModule)), std::move(dRt), - std::move(boost::apply_visitor(deltaVisitor, forgetGateModule))), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, forgetGateModule), dRt, + boost::apply_visitor(deltaVisitor, forgetGateModule)), forgetGateModule); // Put delta zt. @@ -289,10 +284,12 @@ void GRU::Backward( boost::apply_visitor(deltaVisitor, hiddenStateModule); // Get delta ht - 1 for input gate and forget gate. - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, input2GateModule)), - std::move(prevError.submat(0, 0, 2 * outSize - 1, batchSize - 1)), - std::move(boost::apply_visitor(deltaVisitor, output2GateModule))), + arma::mat prevErrorSubview = prevError.submat(0, 0, 2 * outSize - 1, + batchSize - 1); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, input2GateModule), + prevErrorSubview, + boost::apply_visitor(deltaVisitor, output2GateModule)), output2GateModule); // Add delta ht - 1 from hidden state. @@ -301,13 +298,13 @@ void GRU::Backward( boost::apply_visitor(outputParameterVisitor, forgetGateModule); // Add delta ht - 1 from ht. - boost::apply_visitor(deltaVisitor, output2GateModule) += gy % + boost::apply_visitor(deltaVisitor, output2GateModule) += gyLocal % boost::apply_visitor(outputParameterVisitor, inputGateModule); // Get delta input. - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, input2GateModule)), std::move(prevError), - std::move(boost::apply_visitor(deltaVisitor, input2GateModule))), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, input2GateModule), prevError, + boost::apply_visitor(deltaVisitor, input2GateModule)), input2GateModule); backwardStep++; @@ -319,9 +316,9 @@ void GRU::Backward( template template void GRU::Gradient( - arma::Mat&& input, - arma::Mat&& /* error */, - arma::Mat&& /* gradient */) + const arma::Mat& input, + const arma::Mat& /* error */, + arma::Mat& /* gradient */) { if (input.n_cols != batchSize) { @@ -349,19 +346,18 @@ void GRU::Gradient( gradIterator = --(--outParameter.end()); } - boost::apply_visitor(GradientVisitor(std::move(input), std::move(prevError)), - input2GateModule); + boost::apply_visitor(GradientVisitor(input, prevError), input2GateModule); boost::apply_visitor(GradientVisitor( - std::move(*gradIterator), - std::move(prevError.submat(0, 0, 2 * outSize - 1, batchSize - 1))), + *gradIterator, + prevError.submat(0, 0, 2 * outSize - 1, batchSize - 1)), output2GateModule); boost::apply_visitor(GradientVisitor( *gradIterator % boost::apply_visitor(outputParameterVisitor, forgetGateModule), - std::move(prevError.submat(2 * outSize, 0, 3 * outSize - 1, - batchSize - 1))), outputHidden2GateModule); + prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1)), + outputHidden2GateModule); gradIterator--; } diff --git a/src/mlpack/methods/ann/layer/hard_tanh.hpp b/src/mlpack/methods/ann/layer/hard_tanh.hpp index 8ff75b899f..fa5b974e33 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh.hpp @@ -67,7 +67,7 @@ class HardTanH * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -79,9 +79,9 @@ class HardTanH * @param g The calculated gradient. */ template - void Backward(const DataType&& input, - DataType&& gy, - DataType&& g); + void Backward(const DataType& input, + const DataType& gy, + DataType& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp index 1332b89d8d..473330cb07 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp @@ -31,7 +31,7 @@ HardTanH::HardTanH( template template void HardTanH::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { output = input; for (size_t i = 0; i < input.n_elem; i++) @@ -44,7 +44,7 @@ void HardTanH::Forward( template template void HardTanH::Backward( - const DataType&& input, DataType&& gy, DataType&& g) + const DataType& input, const DataType& gy, DataType& g) { g = gy; for (size_t i = 0; i < input.n_elem; i++) diff --git a/src/mlpack/methods/ann/layer/hardshrink.hpp b/src/mlpack/methods/ann/layer/hardshrink.hpp index 170eb21524..aa7f51b550 100644 --- a/src/mlpack/methods/ann/layer/hardshrink.hpp +++ b/src/mlpack/methods/ann/layer/hardshrink.hpp @@ -66,7 +66,7 @@ class HardShrink * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -78,9 +78,9 @@ class HardShrink * @param g The calculated gradient. */ template - void Backward(const DataType&& input, - DataType&& gy, - DataType&& g); + void Backward(const DataType& input, + DataType& gy, + DataType& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/hardshrink_impl.hpp b/src/mlpack/methods/ann/layer/hardshrink_impl.hpp index 669d4f13d2..d1f9112eaa 100644 --- a/src/mlpack/methods/ann/layer/hardshrink_impl.hpp +++ b/src/mlpack/methods/ann/layer/hardshrink_impl.hpp @@ -30,7 +30,7 @@ HardShrink::HardShrink(const double lambda) : template template void HardShrink::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { output = ((input > lambda) + (input < -lambda)) % input; } @@ -38,7 +38,7 @@ void HardShrink::Forward( template template void HardShrink::Backward( - const DataType&& input, DataType&& gy, DataType&& g) + const DataType& input, DataType& gy, DataType& g) { DataType derivative; derivative = (arma::ones(arma::size(input)) - (input == 0)); diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp index f0216b8f0b..fb8e1f9717 100644 --- a/src/mlpack/methods/ann/layer/highway.hpp +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -74,11 +74,6 @@ class Highway //! Destroy the Highway object. ~Highway(); - /** - * Destroy all the modules added to the Highway object. - */ - void DeleteModules(); - /** * Reset the layer parameter. */ @@ -92,7 +87,7 @@ class Highway * @param output Resulting output activation. */ template - void Forward(arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed-backward pass of a neural network, calculating the function @@ -104,9 +99,9 @@ class Highway * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activation. @@ -116,9 +111,9 @@ class Highway * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); /** * Add a new module to the model. @@ -126,14 +121,22 @@ class Highway * @param args The layer parameter. */ template - void Add(Args... args) { network.push_back(new LayerType(args...)); } + void Add(Args... args) + { + network.push_back(new LayerType(args...)); + networkOwnerships.push_back(true); + } /** * Add a new module to the model. * * @param layer The Layer to be added to the model. */ - void Add(LayerTypes layer) { network.push_back(layer); } + void Add(LayerTypes layer) + { + network.push_back(layer); + networkOwnerships.push_back(false); + } //! Return the modules of the model. std::vector >& Model() @@ -190,6 +193,9 @@ class Highway //! Locally-stored network modules. std::vector > network; + //! The list of network modules we are responsible for. + std::vector networkOwnerships; + //! Locally-stored empty list of modules. std::vector > empty; diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp index 87acf9553d..e0b344b608 100644 --- a/src/mlpack/methods/ann/layer/highway_impl.hpp +++ b/src/mlpack/methods/ann/layer/highway_impl.hpp @@ -57,23 +57,10 @@ Highway::~Highway() { if (!model) { - for (LayerTypes& layer : network) + for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(deleteVisitor, layer); - } - } -} - -template -void Highway< - InputDataType, OutputDataType, CustomLayers...>::DeleteModules() -{ - if (model) - { - for (LayerTypes& layer : network) - { - boost::apply_visitor(deleteVisitor, layer); + if (networkOwnerships[i]) + boost::apply_visitor(deleteVisitor, network[i]); } } } @@ -91,10 +78,10 @@ template template void Highway::Forward( - arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, network.front()))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network.front())), network.front()); if (!reset) @@ -121,9 +108,9 @@ void Highway::Forward( boost::apply_visitor(SetInputHeightVisitor(height), network[i]); } - boost::apply_visitor(ForwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[i - 1])), std::move( - boost::apply_visitor(outputParameterVisitor, network[i]))), + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i - 1]), + boost::apply_visitor(outputParameterVisitor, network[i])), network[i]); if (!reset) @@ -167,23 +154,24 @@ template template void Highway::Backward( - const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), - std::move(gy % transformGateActivation), - std::move(boost::apply_visitor(deltaVisitor, network.back()))), + arma::Mat gyTransform = gy % transformGateActivation; + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network.back()), + gyTransform, + boost::apply_visitor(deltaVisitor, network.back())), network.back()); for (size_t i = 2; i < network.size() + 1; ++i) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[network.size() - i])), std::move( - boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])), - std::move(boost::apply_visitor(deltaVisitor, - network[network.size() - i]))), network[network.size() - i]); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]), + boost::apply_visitor(deltaVisitor, + network[network.size() - i])), network[network.size() - i]); } g = boost::apply_visitor(deltaVisitor, network.front()); @@ -198,24 +186,25 @@ template template void Highway::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[network.size() - 2])), - std::move(error % transformGateActivation)), network.back()); + arma::Mat errorTransform = error % transformGateActivation; + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - 2]), + errorTransform), network.back()); for (size_t i = 2; i < network.size(); ++i) { - boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[network.size() - i - 1])), std::move( - boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]))), + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i - 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])), network[network.size() - i]); } - boost::apply_visitor(GradientVisitor(std::move(input), std::move( - boost::apply_visitor(deltaVisitor, network[1]))), network.front()); + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, network[1])), network.front()); gradient.submat(0, 0, transformWeight.n_elem - 1, 0) = arma::vectorise( transformGateError * input.t()); diff --git a/src/mlpack/methods/ann/layer/join.hpp b/src/mlpack/methods/ann/layer/join.hpp index 2f6ecde25f..dcf0c06327 100644 --- a/src/mlpack/methods/ann/layer/join.hpp +++ b/src/mlpack/methods/ann/layer/join.hpp @@ -44,7 +44,7 @@ class Join * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -56,9 +56,9 @@ class Join * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/join_impl.hpp b/src/mlpack/methods/ann/layer/join_impl.hpp index 6ceee0c135..14d85bc88d 100644 --- a/src/mlpack/methods/ann/layer/join_impl.hpp +++ b/src/mlpack/methods/ann/layer/join_impl.hpp @@ -29,7 +29,7 @@ Join::Join() : template template void Join::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { inSizeRows = input.n_rows; inSizeCols = input.n_cols; @@ -39,11 +39,12 @@ void Join::Forward( template template void Join::Backward( - const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - g = arma::mat(gy.memptr(), inSizeRows, inSizeCols, false, false); + g = arma::mat(((arma::Mat&) gy).memptr(), inSizeRows, inSizeCols, false, + false); } template diff --git a/src/mlpack/methods/ann/layer/layer_norm.hpp b/src/mlpack/methods/ann/layer/layer_norm.hpp index f53d35a860..55642b21d2 100644 --- a/src/mlpack/methods/ann/layer/layer_norm.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm.hpp @@ -90,7 +90,7 @@ class LayerNorm * @param output Resulting output activations. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Backward pass through the layer. @@ -100,9 +100,9 @@ class LayerNorm * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activations. @@ -112,9 +112,9 @@ class LayerNorm * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp index b9383d7dd4..eb49377972 100644 --- a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp @@ -57,7 +57,7 @@ void LayerNorm::Reset() template template void LayerNorm::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { mean = arma::mean(input, 0); variance = arma::var(input, 1, 0); @@ -78,7 +78,7 @@ void LayerNorm::Forward( template template void LayerNorm::Backward( - const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) { const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); @@ -102,9 +102,9 @@ void LayerNorm::Backward( template template void LayerNorm::Gradient( - const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient) { gradient.set_size(size + size, 1); diff --git a/src/mlpack/methods/ann/layer/layer_traits.hpp b/src/mlpack/methods/ann/layer/layer_traits.hpp index 1955235f87..665adb9b2b 100644 --- a/src/mlpack/methods/ann/layer/layer_traits.hpp +++ b/src/mlpack/methods/ann/layer/layer_traits.hpp @@ -116,6 +116,10 @@ HAS_MEM_FUNC(Run, HasRunCheck); // can use with SFINAE to catch when a type has a Bias() function. HAS_MEM_FUNC(Bias, HasBiasCheck); +// This gives us a HasMaxIterationsC type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a MaxIterations() function. +HAS_MEM_FUNC(MaxIterations, HasMaxIterations); + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 810c5b6b93..3c17021e45 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -42,6 +42,8 @@ #include #include #include +#include +#include // Convolution modules. #include @@ -243,6 +245,7 @@ using LayerTypes = boost::variant< Padding*, PReLU*, WeightNorm*, + CELU*, MoreTypes, CustomLayers*... >; diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index f9d7b230f9..1780c5a737 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -61,7 +61,7 @@ class LeakyReLU * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -73,7 +73,7 @@ class LeakyReLU * @param g The calculated gradient. */ template - void Backward(const DataType&& input, DataType&& gy, DataType&& g); + void Backward(const DataType& input, const DataType& gy, DataType& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp b/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp index 5d024ad3dd..8053cd031a 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp @@ -30,7 +30,7 @@ LeakyReLU::LeakyReLU( template template void LeakyReLU::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { output = arma::max(input, alpha * input); } @@ -38,7 +38,7 @@ void LeakyReLU::Forward( template template void LeakyReLU::Backward( - const DataType&& input, DataType&& gy, DataType&& g) + const DataType& input, const DataType& gy, DataType& g) { DataType derivative; derivative.set_size(arma::size(input)); diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp index bf4074f61b..8d3c984683 100644 --- a/src/mlpack/methods/ann/layer/linear.hpp +++ b/src/mlpack/methods/ann/layer/linear.hpp @@ -64,7 +64,7 @@ class Linear * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -76,9 +76,9 @@ class Linear * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -88,9 +88,9 @@ class Linear * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/linear_impl.hpp b/src/mlpack/methods/ann/layer/linear_impl.hpp index 127d789716..0ec31f1070 100644 --- a/src/mlpack/methods/ann/layer/linear_impl.hpp +++ b/src/mlpack/methods/ann/layer/linear_impl.hpp @@ -54,7 +54,7 @@ template template void Linear::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { output = weight * input; output.each_col() += bias; @@ -64,7 +64,7 @@ template template void Linear::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = weight.t() * gy; } @@ -73,9 +73,9 @@ template template void Linear::Gradient( - const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise( error * input.t()); diff --git a/src/mlpack/methods/ann/layer/linear_no_bias.hpp b/src/mlpack/methods/ann/layer/linear_no_bias.hpp index 1b2c5ffc02..620bf896b9 100644 --- a/src/mlpack/methods/ann/layer/linear_no_bias.hpp +++ b/src/mlpack/methods/ann/layer/linear_no_bias.hpp @@ -63,7 +63,7 @@ class LinearNoBias * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -75,9 +75,9 @@ class LinearNoBias * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -87,9 +87,9 @@ class LinearNoBias * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp b/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp index 41e432da17..fee30ec783 100644 --- a/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp +++ b/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp @@ -52,7 +52,7 @@ template template void LinearNoBias::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { output = weight * input; } @@ -61,7 +61,7 @@ template template void LinearNoBias::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = weight.t() * gy; } @@ -70,9 +70,9 @@ template template void LinearNoBias::Gradient( - const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise( error * input.t()); diff --git a/src/mlpack/methods/ann/layer/log_softmax.hpp b/src/mlpack/methods/ann/layer/log_softmax.hpp index 613eac4aa3..25a9495d9d 100644 --- a/src/mlpack/methods/ann/layer/log_softmax.hpp +++ b/src/mlpack/methods/ann/layer/log_softmax.hpp @@ -49,7 +49,7 @@ class LogSoftMax * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -61,9 +61,9 @@ class LogSoftMax * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/log_softmax_impl.hpp b/src/mlpack/methods/ann/layer/log_softmax_impl.hpp index d4ca9c53ae..2c6ea6c635 100644 --- a/src/mlpack/methods/ann/layer/log_softmax_impl.hpp +++ b/src/mlpack/methods/ann/layer/log_softmax_impl.hpp @@ -27,7 +27,7 @@ LogSoftMax::LogSoftMax() template template void LogSoftMax::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { arma::mat maxInput = arma::repmat(arma::max(input), input.n_rows, 1); output = (maxInput - input); @@ -64,9 +64,9 @@ void LogSoftMax::Forward( template template void LogSoftMax::Backward( - const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { g = arma::exp(input) + gy; } diff --git a/src/mlpack/methods/ann/layer/lookup.hpp b/src/mlpack/methods/ann/layer/lookup.hpp index 59e5547cda..ac5ced3ca4 100644 --- a/src/mlpack/methods/ann/layer/lookup.hpp +++ b/src/mlpack/methods/ann/layer/lookup.hpp @@ -52,7 +52,7 @@ class Lookup * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -64,9 +64,9 @@ class Lookup * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - const arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -76,9 +76,9 @@ class Lookup * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/lookup_impl.hpp b/src/mlpack/methods/ann/layer/lookup_impl.hpp index 3c4b8e96c3..2dde378e8a 100644 --- a/src/mlpack/methods/ann/layer/lookup_impl.hpp +++ b/src/mlpack/methods/ann/layer/lookup_impl.hpp @@ -32,7 +32,7 @@ Lookup::Lookup( template template void Lookup::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { output = weights.cols(arma::conv_to::from(input) - 1); } @@ -40,9 +40,9 @@ void Lookup::Forward( template template void Lookup::Backward( - const arma::Mat&& /* input */, - const arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy; } @@ -50,9 +50,9 @@ void Lookup::Backward( template template void Lookup::Gradient( - const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { gradient = arma::zeros >(weights.n_rows, weights.n_cols); gradient.cols(arma::conv_to::from(input) - 1) = error; diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 9c1f75c715..20a4b07248 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -84,7 +84,7 @@ class LSTM * @param output Resulting output activation. */ template - void Forward(InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed-forward pass of a neural network, evaluating the function @@ -96,9 +96,9 @@ class LSTM * @param useCellState Use the cellState passed in the LSTM cell. */ template - void Forward(InputType&& input, - OutputType&& output, - OutputType&& cellState, + void Forward(const InputType& input, + OutputType& output, + OutputType& cellState, bool useCellState = false); /** @@ -111,9 +111,9 @@ class LSTM * @param g The calculated gradient. */ template - void Backward(const InputType&& input, - ErrorType&& gy, - GradientType&& g); + void Backward(const InputType& input, + const ErrorType& gy, + GradientType& g); /* * Reset the layer parameter. @@ -136,9 +136,9 @@ class LSTM * @param gradient The calculated gradient. */ template - void Gradient(InputType&& input, - ErrorType&& error, - GradientType&& gradient); + void Gradient(const InputType& input, + const ErrorType& error, + GradientType& gradient); //! Get the maximum number of steps to backpropagate through time (BPTT). size_t Rho() const { return rho; } diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index d46ef43e00..4b4dd03e91 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -163,19 +163,19 @@ void LSTM::Reset() template template void LSTM::Forward( - InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { //! Locally-stored cellState. OutputType cellState; - Forward(std::move(input), std::move(output), std::move(cellState), false); + Forward(input, output, cellState, false); } // Forward when cellState is needed overloaded LSTM::Forward(). template template -void LSTM::Forward(InputType&& input, - OutputType&& output, - OutputType&& cellState, +void LSTM::Forward(const InputType& input, + OutputType& output, + OutputType& cellState, bool useCellState) { // Check if the batch size changed, the number of cols is defines the input @@ -288,20 +288,27 @@ void LSTM::Forward(InputType&& input, template template void LSTM::Backward( - const InputType&& /* input */, ErrorType&& gy, GradientType&& g) + const InputType& /* input */, const ErrorType& gy, GradientType& g) { + ErrorType gyLocal; if (gradientStepIdx > 0) { - gy += prevError; + gyLocal = gy + prevError; + } + else + { + // Make an alias. + gyLocal = ErrorType(((ErrorType&) gy).memptr(), gy.n_rows, gy.n_cols, false, + false); } outputGateError = - gy % cellActivation.cols(backwardStep - batchStep, backwardStep) % + gyLocal % cellActivation.cols(backwardStep - batchStep, backwardStep) % (outputGateActivation.cols(backwardStep - batchStep, backwardStep) % (1.0 - outputGateActivation.cols(backwardStep - batchStep, backwardStep))); - OutputDataType cellError = gy % + OutputDataType cellError = gyLocal % outputGateActivation.cols(backwardStep - batchStep, backwardStep) % (1 - arma::pow(cellActivation.cols(backwardStep - batchStep, backwardStep), 2)) + outputGateError.each_col() % @@ -359,7 +366,9 @@ void LSTM::Backward( template template void LSTM::Gradient( - InputType&& input, ErrorType&& /* error */, GradientType&& gradient) + const InputType& input, + const ErrorType& /* error */, + GradientType& gradient) { // Input2GateOutputWeight and input2GateOutputBias gradients. gradient.submat(0, 0, input2GateOutputWeight.n_elem - 1, 0) = diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index e0793c2ba2..3f3af81949 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -78,7 +78,7 @@ class MaxPooling * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, using 3rd-order tensors as @@ -90,9 +90,9 @@ class MaxPooling * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. const OutputDataType& OutputParameter() const { return outputParameter; } @@ -180,10 +180,10 @@ class MaxPooling arma::Mat& poolingIndices) { for (size_t j = 0, colidx = 0; j < output.n_cols; - ++j, colidx += strideWidth) + ++j, colidx += strideHeight) { for (size_t i = 0, rowidx = 0; i < output.n_rows; - ++i, rowidx += strideHeight) + ++i, rowidx += strideWidth) { arma::mat subInput = input( arma::span(rowidx, rowidx + kernelWidth - 1 - offset), diff --git a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp index e89b295f5b..69722466c3 100644 --- a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp @@ -54,11 +54,11 @@ MaxPooling::MaxPooling( template template void MaxPooling::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; inSize = input.n_elem / (inputWidth * inputHeight * batchSize); - inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputTemp = arma::cube(const_cast&>(input).memptr(), inputWidth, inputHeight, batchSize * inSize, false, false); if (floor) @@ -122,10 +122,10 @@ void MaxPooling::Forward( template template void MaxPooling::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { - arma::cube mappedError = arma::cube(gy.memptr(), outputWidth, - outputHeight, outSize, false, false); + arma::cube mappedError = arma::cube(((arma::Mat&) gy).memptr(), + outputWidth, outputHeight, outSize, false, false); gTemp = arma::zeros(inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices); diff --git a/src/mlpack/methods/ann/layer/mean_pooling.hpp b/src/mlpack/methods/ann/layer/mean_pooling.hpp index 5894e0ab65..34c3a811e2 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling.hpp @@ -58,7 +58,7 @@ class MeanPooling * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, using 3rd-order tensors as @@ -70,9 +70,9 @@ class MeanPooling * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } @@ -156,9 +156,6 @@ class MeanPooling template void Pooling(const arma::Mat& input, arma::Mat& output) { - const size_t rStep = kernelWidth; - const size_t cStep = kernelHeight; - for (size_t j = 0, colidx = 0; j < output.n_cols; ++j, colidx += strideHeight) { @@ -166,8 +163,8 @@ class MeanPooling ++i, rowidx += strideWidth) { arma::mat subInput = input( - arma::span(rowidx, rowidx + rStep - 1 - offset), - arma::span(colidx, colidx + cStep - 1 - offset)); + arma::span(rowidx, rowidx + kernelWidth - 1 - offset), + arma::span(colidx, colidx + kernelHeight - 1 - offset)); output(i, j) = arma::mean(arma::mean(subInput)); } diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index 2c6d5d1da9..a9b6a4d1c9 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -54,11 +54,11 @@ MeanPooling::MeanPooling( template template void MeanPooling::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; inSize = input.n_elem / (inputWidth * inputHeight * batchSize); - inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputTemp = arma::cube(const_cast&>(input).memptr(), inputWidth, inputHeight, batchSize * inSize, false, false); if (floor) @@ -97,12 +97,12 @@ void MeanPooling::Forward( template template void MeanPooling::Backward( - const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - arma::cube mappedError = arma::cube(gy.memptr(), outputWidth, - outputHeight, outSize, false, false); + arma::cube mappedError = arma::cube(((arma::Mat&) gy).memptr(), + outputWidth, outputHeight, outSize, false, false); gTemp = arma::zeros(inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices); diff --git a/src/mlpack/methods/ann/layer/minibatch_discrimination.hpp b/src/mlpack/methods/ann/layer/minibatch_discrimination.hpp index 4f995998bc..b9f258091e 100644 --- a/src/mlpack/methods/ann/layer/minibatch_discrimination.hpp +++ b/src/mlpack/methods/ann/layer/minibatch_discrimination.hpp @@ -81,7 +81,7 @@ class MiniBatchDiscrimination * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed-backward pass of a neural network, calculating the function @@ -93,9 +93,9 @@ class MiniBatchDiscrimination * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activation. @@ -105,9 +105,9 @@ class MiniBatchDiscrimination * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& /* error */, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& /* error */, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/minibatch_discrimination_impl.hpp b/src/mlpack/methods/ann/layer/minibatch_discrimination_impl.hpp index 7409a5c4d9..18f38d5da3 100644 --- a/src/mlpack/methods/ann/layer/minibatch_discrimination_impl.hpp +++ b/src/mlpack/methods/ann/layer/minibatch_discrimination_impl.hpp @@ -52,7 +52,7 @@ void MiniBatchDiscrimination::Reset() template template void MiniBatchDiscrimination::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; tempM = weight * input; @@ -88,7 +88,7 @@ void MiniBatchDiscrimination::Forward( template template void MiniBatchDiscrimination::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = gy.head_rows(A); arma::Mat gM = gy.tail_rows(B); @@ -117,9 +117,9 @@ void MiniBatchDiscrimination::Backward( template template void MiniBatchDiscrimination::Gradient( - const arma::Mat&& input, - arma::Mat&& /* error */, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& /* error */, + arma::Mat& gradient) { gradient = arma::vectorise(deltaTemp * input.t()); } diff --git a/src/mlpack/methods/ann/layer/multiply_constant.hpp b/src/mlpack/methods/ann/layer/multiply_constant.hpp index 98cba880b9..9e020eb87f 100644 --- a/src/mlpack/methods/ann/layer/multiply_constant.hpp +++ b/src/mlpack/methods/ann/layer/multiply_constant.hpp @@ -47,7 +47,7 @@ class MultiplyConstant * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network. The backward pass @@ -58,7 +58,7 @@ class MultiplyConstant * @param g The calculated gradient. */ template - void Backward(const DataType&& /* input */, DataType&& gy, DataType&& g); + void Backward(const DataType& /* input */, const DataType& gy, DataType& g); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/multiply_constant_impl.hpp b/src/mlpack/methods/ann/layer/multiply_constant_impl.hpp index 87d0007a93..648e296602 100644 --- a/src/mlpack/methods/ann/layer/multiply_constant_impl.hpp +++ b/src/mlpack/methods/ann/layer/multiply_constant_impl.hpp @@ -29,7 +29,7 @@ MultiplyConstant::MultiplyConstant( template template void MultiplyConstant::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { output = input * scalar; } @@ -37,7 +37,7 @@ void MultiplyConstant::Forward( template template void MultiplyConstant::Backward( - const DataType&& /* input */, DataType&& gy, DataType&& g) + const DataType& /* input */, const DataType& gy, DataType& g) { g = gy * scalar; } diff --git a/src/mlpack/methods/ann/layer/multiply_merge.hpp b/src/mlpack/methods/ann/layer/multiply_merge.hpp index a247521c73..36430ea05d 100644 --- a/src/mlpack/methods/ann/layer/multiply_merge.hpp +++ b/src/mlpack/methods/ann/layer/multiply_merge.hpp @@ -61,7 +61,7 @@ class MultiplyMerge * @param output Resulting output activation. */ template - void Forward(InputType&& /* input */, OutputType&& output); + void Forward(const InputType& /* input */, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -73,9 +73,9 @@ class MultiplyMerge * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -85,9 +85,9 @@ class MultiplyMerge * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); /* * Add a new module to the model. diff --git a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp index 12ff210292..174bddfd2f 100644 --- a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp +++ b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp @@ -47,14 +47,14 @@ template template void MultiplyMerge::Forward( - InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { if (run) { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, network[i]))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network[i])), network[i]); } } @@ -70,15 +70,15 @@ template template void MultiplyMerge::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { if (run) { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[i])), std::move(gy), std::move( - boost::apply_visitor(deltaVisitor, network[i]))), network[i]); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i]), gy, + boost::apply_visitor(deltaVisitor, network[i])), network[i]); } g = boost::apply_visitor(deltaVisitor, network[0]); @@ -95,16 +95,15 @@ template template void MultiplyMerge::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */ ) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */ ) { if (run) { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), - network[i]); + boost::apply_visitor(GradientVisitor(input, error), network[i]); } } } diff --git a/src/mlpack/methods/ann/layer/padding.hpp b/src/mlpack/methods/ann/layer/padding.hpp index 0469f2e931..7ebcab07a2 100644 --- a/src/mlpack/methods/ann/layer/padding.hpp +++ b/src/mlpack/methods/ann/layer/padding.hpp @@ -55,7 +55,7 @@ class Padding * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -67,9 +67,9 @@ class Padding * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - const arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/padding_impl.hpp b/src/mlpack/methods/ann/layer/padding_impl.hpp index 8578e7ca09..6daa98321b 100644 --- a/src/mlpack/methods/ann/layer/padding_impl.hpp +++ b/src/mlpack/methods/ann/layer/padding_impl.hpp @@ -38,7 +38,7 @@ Padding::Padding( template template void Padding::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { nRows = input.n_rows; nCols = input.n_cols; @@ -51,9 +51,9 @@ void Padding::Forward( template template void Padding::Backward( - const arma::Mat&& /* input */, - const arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy.submat(padWLeft, padHTop, padWLeft + nRows - 1, padHTop + nCols - 1); diff --git a/src/mlpack/methods/ann/layer/parametric_relu.hpp b/src/mlpack/methods/ann/layer/parametric_relu.hpp index 9edde60190..6e66f185c1 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu.hpp @@ -68,7 +68,7 @@ class PReLU * @param output Resulting output activation. */ template - void Forward(const InputType&& input, OutputType&& output); + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -80,7 +80,7 @@ class PReLU * @param g The calculated gradient. */ template - void Backward(const DataType&& input, DataType&& gy, DataType&& g); + void Backward(const DataType& input, const DataType& gy, DataType& g); /** * Calculate the gradient using the output delta and the input activation. @@ -90,9 +90,9 @@ class PReLU * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return alpha; } diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp index d7c8ac4ffb..a794d99eae 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -39,7 +39,7 @@ void PReLU::Reset() template template void PReLU::Forward( - const InputType&& input, OutputType&& output) + const InputType& input, OutputType& output) { output = input; arma::uvec negative = arma::find(input < 0); @@ -49,7 +49,7 @@ void PReLU::Forward( template template void PReLU::Backward( - const DataType&& input, DataType&& gy, DataType&& g) + const DataType& input, const DataType& gy, DataType& g) { DataType derivative; derivative.set_size(arma::size(input)); @@ -64,8 +64,9 @@ void PReLU::Backward( template template void PReLU::Gradient( - const arma::Mat&& input, arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { if (gradient.n_elem == 0) { diff --git a/src/mlpack/methods/ann/layer/recurrent.hpp b/src/mlpack/methods/ann/layer/recurrent.hpp index fc1914352a..da677cad5a 100644 --- a/src/mlpack/methods/ann/layer/recurrent.hpp +++ b/src/mlpack/methods/ann/layer/recurrent.hpp @@ -50,9 +50,6 @@ class Recurrent */ Recurrent(); - //! Destructor to release allocated memory. - ~Recurrent(); - //! Copy constructor. Recurrent(const Recurrent&); @@ -83,7 +80,7 @@ class Recurrent * @param output Resulting output activation. */ template - void Forward(arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -95,9 +92,9 @@ class Recurrent * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -107,9 +104,9 @@ class Recurrent * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */); //! Get the model modules. std::vector >& Model() { return network; } diff --git a/src/mlpack/methods/ann/layer/recurrent_attention.hpp b/src/mlpack/methods/ann/layer/recurrent_attention.hpp index 087b6bdb6b..3c43d75d0e 100644 --- a/src/mlpack/methods/ann/layer/recurrent_attention.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_attention.hpp @@ -35,11 +35,12 @@ namespace ann /** Artificial Neural Network. */ { * * @code * @article{MnihHGK14, - * title={Recurrent Models of Visual Attention}, - * author={Volodymyr Mnih, Nicolas Heess, Alex Graves, Koray Kavukcuoglu}, - * journal={CoRR}, - * volume={abs/1406.6247}, - * year={2014} + * title = {Recurrent Models of Visual Attention}, + * author = {Volodymyr Mnih, Nicolas Heess, Alex Graves, Koray Kavukcuoglu}, + * journal = {CoRR}, + * volume = {abs/1406.6247}, + * year = {2014}, + * url = {https://arxiv.org/abs/1406.6247} * } * @endcode * @@ -64,9 +65,9 @@ class RecurrentAttention /** * Create the RecurrentAttention object using the specified modules. * - * @param start The module output size. - * @param start The recurrent neural network module. - * @param start The action module. + * @param outSize The module output size. + * @param rnn The recurrent neural network module. + * @param action The action module. * @param rho Maximum number of steps to backpropagate through time (BPTT). */ template @@ -83,7 +84,7 @@ class RecurrentAttention * @param output Resulting output activation. */ template - void Forward(arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -95,9 +96,9 @@ class RecurrentAttention * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -107,9 +108,9 @@ class RecurrentAttention * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& /* input */, - arma::Mat&& /* error */, - arma::Mat&& /* gradient */); + void Gradient(const arma::Mat& /* input */, + const arma::Mat& /* error */, + arma::Mat& /* gradient */); //! Get the model modules. std::vector>& Model() { return network; } @@ -154,19 +155,19 @@ class RecurrentAttention // Gradient of the action module. if (backwardStep == (rho - 1)) { - boost::apply_visitor(GradientVisitor(std::move(initialInput), - std::move(actionError)), actionModule); + boost::apply_visitor(GradientVisitor(initialInput, actionError), + actionModule); } else { - boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, actionModule)), std::move(actionError)), + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, actionModule), actionError), actionModule); } // Gradient of the recurrent module. - boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, rnnModule)), std::move(recurrentError)), + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, rnnModule), recurrentError), rnnModule); attentionGradient += intermediateGradient; diff --git a/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp index b29e369828..a276ed20c0 100644 --- a/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp @@ -58,7 +58,7 @@ RecurrentAttention::RecurrentAttention( template template void RecurrentAttention::Forward( - arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { // Initialize the action input. if (initialInput.is_empty()) @@ -71,15 +71,15 @@ void RecurrentAttention::Forward( { if (forwardStep == 0) { - boost::apply_visitor(ForwardVisitor(std::move(initialInput), std::move( - boost::apply_visitor(outputParameterVisitor, actionModule))), + boost::apply_visitor(ForwardVisitor(initialInput, + boost::apply_visitor(outputParameterVisitor, actionModule)), actionModule); } else { - boost::apply_visitor(ForwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, rnnModule)), std::move(boost::apply_visitor( - outputParameterVisitor, actionModule))), actionModule); + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, rnnModule), boost::apply_visitor( + outputParameterVisitor, actionModule)), actionModule); } // Initialize the glimpse input. @@ -89,8 +89,8 @@ void RecurrentAttention::Forward( actionModule).n_elem - 1, 1) = boost::apply_visitor( outputParameterVisitor, actionModule); - boost::apply_visitor(ForwardVisitor(std::move(glimpseInput), - std::move(boost::apply_visitor(outputParameterVisitor, rnnModule))), + boost::apply_visitor(ForwardVisitor(glimpseInput, + boost::apply_visitor(outputParameterVisitor, rnnModule)), rnnModule); // Save the output parameter when training the module. @@ -99,7 +99,7 @@ void RecurrentAttention::Forward( for (size_t l = 0; l < network.size(); ++l) { boost::apply_visitor(SaveOutputParameterVisitor( - std::move(moduleOutputParameter)), network[l]); + moduleOutputParameter), network[l]); } } } @@ -113,9 +113,9 @@ void RecurrentAttention::Forward( template template void RecurrentAttention::Backward( - const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { if (intermediateGradient.is_empty() && backwardStep == 0) { @@ -137,9 +137,9 @@ void RecurrentAttention::Backward( { size_t offset = 0; offset += boost::apply_visitor(GradientSetVisitor( - std::move(intermediateGradient), offset), rnnModule); + intermediateGradient, offset), rnnModule); boost::apply_visitor(GradientSetVisitor( - std::move(intermediateGradient), offset), actionModule); + intermediateGradient, offset), actionModule); attentionGradient.zeros(); } @@ -159,24 +159,24 @@ void RecurrentAttention::Backward( for (size_t l = 0; l < network.size(); ++l) { boost::apply_visitor(LoadOutputParameterVisitor( - std::move(moduleOutputParameter)), network[network.size() - 1 - l]); + moduleOutputParameter), network[network.size() - 1 - l]); } if (backwardStep == (rho - 1)) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, actionModule)), std::move(actionError), - std::move(actionDelta)), actionModule); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, actionModule), actionError, + actionDelta), actionModule); } else { - boost::apply_visitor(BackwardVisitor(std::move(initialInput), - std::move(actionError), std::move(actionDelta)), actionModule); + boost::apply_visitor(BackwardVisitor(initialInput, actionError, + actionDelta), actionModule); } - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, rnnModule)), std::move(recurrentError), - std::move(rnnDelta)), rnnModule); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, rnnModule), recurrentError, rnnDelta), + rnnModule); if (backwardStep == 0) { @@ -194,15 +194,15 @@ void RecurrentAttention::Backward( template template void RecurrentAttention::Gradient( - arma::Mat&& /* input */, - arma::Mat&& /* error */, - arma::Mat&& /* gradient */) + const arma::Mat& /* input */, + const arma::Mat& /* error */, + arma::Mat& /* gradient */) { size_t offset = 0; offset += boost::apply_visitor(GradientUpdateVisitor( - std::move(attentionGradient), offset), rnnModule); + attentionGradient, offset), rnnModule); boost::apply_visitor(GradientUpdateVisitor( - std::move(attentionGradient), offset), actionModule); + attentionGradient, offset), actionModule); } template diff --git a/src/mlpack/methods/ann/layer/recurrent_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_impl.hpp index baa24f41b9..e526baef35 100644 --- a/src/mlpack/methods/ann/layer/recurrent_impl.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_impl.hpp @@ -37,19 +37,6 @@ Recurrent::Recurrent() : // Nothing to do. } -template -Recurrent::~Recurrent() -{ - if (ownsLayer) - { - boost::apply_visitor(DeleteVisitor(), recurrentModule); - boost::apply_visitor(DeleteVisitor(), initialModule); - boost::apply_visitor(DeleteVisitor(), startModule); - network.clear(); - } -} - template template< @@ -76,8 +63,8 @@ Recurrent::Recurrent( ownsLayer(true) { initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false, false); - recurrentModule = new Sequential<>(false); + mergeModule = new AddMerge<>(false, false, false); + recurrentModule = new Sequential<>(false, false); boost::apply_visitor(AddVisitor(inputModule), initialModule); @@ -116,8 +103,8 @@ Recurrent::Recurrent( feedbackModule = boost::apply_visitor(copyVisitor, network.feedbackModule); transferModule = boost::apply_visitor(copyVisitor, network.transferModule); initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false, false); - recurrentModule = new Sequential<>(false); + mergeModule = new AddMerge<>(false, false, false); + recurrentModule = new Sequential<>(false, false); boost::apply_visitor(AddVisitor(inputModule), initialModule); @@ -143,26 +130,24 @@ template template void Recurrent::Forward( - arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { if (forwardStep == 0) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move(output)), - initialModule); + boost::apply_visitor(ForwardVisitor(input, output), initialModule); } else { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, inputModule))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, inputModule)), inputModule); - boost::apply_visitor(ForwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, transferModule)), std::move( - boost::apply_visitor(outputParameterVisitor, feedbackModule))), + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, transferModule), + boost::apply_visitor(outputParameterVisitor, feedbackModule)), feedbackModule); - boost::apply_visitor(ForwardVisitor(std::move(input), std::move(output)), - recurrentModule); + boost::apply_visitor(ForwardVisitor(input, output), recurrentModule); } output = boost::apply_visitor(outputParameterVisitor, transferModule); @@ -190,7 +175,7 @@ template template void Recurrent::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { if (!recurrentError.is_empty()) { @@ -203,26 +188,26 @@ void Recurrent::Backward( if (backwardStep < (rho - 1)) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, recurrentModule)), std::move(recurrentError), - std::move(boost::apply_visitor(deltaVisitor, recurrentModule))), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, recurrentModule), recurrentError, + boost::apply_visitor(deltaVisitor, recurrentModule)), recurrentModule); - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, inputModule)), std::move( - boost::apply_visitor(deltaVisitor, recurrentModule)), std::move(g)), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, inputModule), + boost::apply_visitor(deltaVisitor, recurrentModule), g), inputModule); - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, feedbackModule)), std::move( - boost::apply_visitor(deltaVisitor, recurrentModule)), std::move( - boost::apply_visitor(deltaVisitor, feedbackModule))), feedbackModule); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, feedbackModule), + boost::apply_visitor(deltaVisitor, recurrentModule), + boost::apply_visitor(deltaVisitor, feedbackModule)), feedbackModule); } else { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, initialModule)), std::move(recurrentError), - std::move(g)), initialModule); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, initialModule), recurrentError, g), + initialModule); } recurrentError = boost::apply_visitor(deltaVisitor, feedbackModule); @@ -233,22 +218,21 @@ template template void Recurrent::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */) { if (gradientStep < (rho - 1)) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), - recurrentModule); + boost::apply_visitor(GradientVisitor(input, error), recurrentModule); - boost::apply_visitor(GradientVisitor(std::move(input), std::move( - boost::apply_visitor(deltaVisitor, mergeModule))), inputModule); + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, mergeModule)), inputModule); - boost::apply_visitor(GradientVisitor(std::move( + boost::apply_visitor(GradientVisitor( feedbackOutputParameter[feedbackOutputParameter.size() - 2 - - gradientStep]), std::move(boost::apply_visitor(deltaVisitor, - mergeModule))), feedbackModule); + gradientStep], boost::apply_visitor(deltaVisitor, + mergeModule)), feedbackModule); } else { @@ -256,8 +240,8 @@ void Recurrent::Gradient( boost::apply_visitor(GradientZeroVisitor(), inputModule); boost::apply_visitor(GradientZeroVisitor(), feedbackModule); - boost::apply_visitor(GradientVisitor(std::move(input), std::move( - boost::apply_visitor(deltaVisitor, startModule))), initialModule); + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, startModule)), initialModule); } gradientStep++; @@ -295,8 +279,8 @@ void Recurrent::serialize( if (Archive::is_loading::value) { initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false, false); - recurrentModule = new Sequential<>(false); + mergeModule = new AddMerge<>(false, false, false); + recurrentModule = new Sequential<>(false, false); boost::apply_visitor(AddVisitor(inputModule), initialModule); diff --git a/src/mlpack/methods/ann/layer/reinforce_normal.hpp b/src/mlpack/methods/ann/layer/reinforce_normal.hpp index bb0ea2e737..c26bc8f495 100644 --- a/src/mlpack/methods/ann/layer/reinforce_normal.hpp +++ b/src/mlpack/methods/ann/layer/reinforce_normal.hpp @@ -49,7 +49,7 @@ class ReinforceNormal * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -61,7 +61,7 @@ class ReinforceNormal * @param g The calculated gradient. */ template - void Backward(const DataType&& input, DataType&& /* gy */, DataType&& g); + void Backward(const DataType& input, const DataType& /* gy */, DataType& g); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } @@ -87,7 +87,7 @@ class ReinforceNormal * Serialize the layer */ template - void serialize(Archive& /* ar */, const unsigned int /* version */); + void serialize(Archive& ar, const unsigned int /* version */); private: //! Standard deviation used during the forward and backward pass. diff --git a/src/mlpack/methods/ann/layer/reinforce_normal_impl.hpp b/src/mlpack/methods/ann/layer/reinforce_normal_impl.hpp index dbe95e00bc..a0a2b03d4f 100644 --- a/src/mlpack/methods/ann/layer/reinforce_normal_impl.hpp +++ b/src/mlpack/methods/ann/layer/reinforce_normal_impl.hpp @@ -29,7 +29,7 @@ ReinforceNormal::ReinforceNormal( template template void ReinforceNormal::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { if (!deterministic) { @@ -49,7 +49,7 @@ void ReinforceNormal::Forward( template template void ReinforceNormal::Backward( - const DataType&& input, DataType&& /* gy */, DataType&& g) + const DataType& input, const DataType& /* gy */, DataType& g) { g = (input - moduleInputParameter.back()) / std::pow(stdev, 2.0); @@ -63,9 +63,9 @@ void ReinforceNormal::Backward( template template void ReinforceNormal::serialize( - Archive& /* ar */, const unsigned int /* version */) + Archive& ar, const unsigned int /* version */) { - // Nothing to do here. + ar & BOOST_SERIALIZATION_NVP(stdev); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 1fc299e2ba..52049b7bf1 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -39,7 +39,8 @@ namespace ann /** Artificial Neural Network. */ { * Xavier Glorot, Matthew Botvinick, Shakir Mohamed and * Alexander Lerchner | Google DeepMind}, * journal = {2017 International Conference on Learning Representations(ICLR)}, - * year = {2017} + * year = {2017}, + * url = {https://deepmind.com/research/publications/beta-VAE-Learning-Basic-Visual-Concepts-with-a-Constrained-Variational-Framework} * } * @endcode * @@ -79,7 +80,7 @@ class Reparametrization * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -91,9 +92,9 @@ class Reparametrization * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp index 21944725d3..6139ca1a86 100644 --- a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -50,7 +50,7 @@ Reparametrization::Reparametrization( template template void Reparametrization::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { if (input.n_rows != 2 * latentSize) { @@ -74,7 +74,7 @@ void Reparametrization::Forward( template template void Reparametrization::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { SoftplusFunction::Deriv(preStdDev, g); diff --git a/src/mlpack/methods/ann/layer/select.hpp b/src/mlpack/methods/ann/layer/select.hpp index 4ec71b9e16..e7c2b593ff 100644 --- a/src/mlpack/methods/ann/layer/select.hpp +++ b/src/mlpack/methods/ann/layer/select.hpp @@ -48,7 +48,7 @@ class Select * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -60,9 +60,9 @@ class Select * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/select_impl.hpp b/src/mlpack/methods/ann/layer/select_impl.hpp index f3a8b6797c..21b85c8fa5 100644 --- a/src/mlpack/methods/ann/layer/select_impl.hpp +++ b/src/mlpack/methods/ann/layer/select_impl.hpp @@ -31,7 +31,7 @@ Select::Select( template template void Select::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { if (elements == 0) { @@ -46,9 +46,9 @@ void Select::Forward( template template void Select::Backward( - const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { if (elements == 0) { diff --git a/src/mlpack/methods/ann/layer/sequential.hpp b/src/mlpack/methods/ann/layer/sequential.hpp index 57ec6742e0..3b84fa9bcd 100644 --- a/src/mlpack/methods/ann/layer/sequential.hpp +++ b/src/mlpack/methods/ann/layer/sequential.hpp @@ -78,6 +78,15 @@ class Sequential */ Sequential(const bool model = true); + /** + * Create the Sequential object using the specified parameters. + * + * @param model Expose all the network modules. + * @param ownsLayers If true, then this module will delete its layers when + * deallocated. + */ + Sequential(const bool model, const bool ownsLayers); + //! Destroy the Sequential object. ~Sequential(); @@ -89,7 +98,7 @@ class Sequential * @param output Resulting output activation. */ template - void Forward(arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, using 3rd-order tensors as @@ -101,9 +110,9 @@ class Sequential * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -113,9 +122,9 @@ class Sequential * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */); /* * Add a new module to the model. @@ -132,11 +141,6 @@ class Sequential */ void Add(LayerTypes layer) { network.push_back(layer); } - /* - * Destroy all the modules added to the Sequential object. - */ - void DeleteModules(); - //! Return the model modules. std::vector >& Model() { @@ -227,6 +231,9 @@ class Sequential //! The input height. size_t height; + + //! Whether we are responsible for deleting the layers held in this module. + bool ownsLayers; }; // class Sequential /* @@ -243,6 +250,25 @@ using Residual = Sequential< } // namespace ann } // namespace mlpack +//! Set the serialization version of the Sequential class. +namespace boost { +namespace serialization { + +template < + typename InputDataType, + typename OutputDataType, + bool Residual, + typename... CustomLayers +> +struct version> +{ + BOOST_STATIC_CONSTANT(int, value = 1); +}; + +} // namespace serialization +} // namespace boost + // Include implementation. #include "sequential_impl.hpp" diff --git a/src/mlpack/methods/ann/layer/sequential_impl.hpp b/src/mlpack/methods/ann/layer/sequential_impl.hpp index 268a21b974..994d377b4c 100644 --- a/src/mlpack/methods/ann/layer/sequential_impl.hpp +++ b/src/mlpack/methods/ann/layer/sequential_impl.hpp @@ -27,9 +27,18 @@ namespace ann /** Artificial Neural Network. */ { template -Sequential< - InputDataType, OutputDataType, Residual, CustomLayers...>::Sequential( - const bool model) : model(model), reset(false), width(0), height(0) +Sequential:: +Sequential(const bool model) : + model(model), reset(false), width(0), height(0), ownsLayers(!model) +{ + // Nothing to do here. +} + +template +Sequential:: +Sequential(const bool model, const bool ownsLayers) : + model(model), reset(false), width(0), height(0), ownsLayers(ownsLayers) { // Nothing to do here. } @@ -39,7 +48,7 @@ template ::~Sequential() { - if (!model) + if (!model && ownsLayers) { for (LayerTypes& layer : network) boost::apply_visitor(deleteVisitor, layer); @@ -49,12 +58,11 @@ Sequential< template template -void Sequential< - InputDataType, OutputDataType, Residual, CustomLayers...>::Forward( - arma::Mat&& input, arma::Mat&& output) +void Sequential:: +Forward(const arma::Mat& input, arma::Mat& output) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, network.front()))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network.front())), network.front()); if (!reset) @@ -81,9 +89,9 @@ void Sequential< boost::apply_visitor(SetInputHeightVisitor(height), network[i]); } - boost::apply_visitor(ForwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[i - 1])), std::move( - boost::apply_visitor(outputParameterVisitor, network[i]))), + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i - 1]), + boost::apply_visitor(outputParameterVisitor, network[i])), network[i]); if (!reset) @@ -126,22 +134,22 @@ template void Sequential< InputDataType, OutputDataType, Residual, CustomLayers...>::Backward( - const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), std::move(gy), - std::move(boost::apply_visitor(deltaVisitor, network.back()))), + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network.back()), gy, + boost::apply_visitor(deltaVisitor, network.back())), network.back()); for (size_t i = 2; i < network.size() + 1; ++i) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[network.size() - i])), std::move( - boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])), - std::move(boost::apply_visitor(deltaVisitor, - network[network.size() - i]))), network[network.size() - i]); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i])), + network[network.size() - i]); } g = boost::apply_visitor(deltaVisitor, network.front()); @@ -155,40 +163,25 @@ void Sequential< template template -void Sequential< - InputDataType, OutputDataType, Residual, CustomLayers...>::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& /* gradient */) +void Sequential:: +Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */) { - boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[network.size() - 2])), std::move(error)), + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - 2]), error), network.back()); for (size_t i = 2; i < network.size(); ++i) { - boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[network.size() - i - 1])), std::move( - boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]))), + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i - 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])), network[network.size() - i]); } - boost::apply_visitor(GradientVisitor(std::move(input), std::move( - boost::apply_visitor(deltaVisitor, network[1]))), network.front()); -} - -template -void Sequential< - InputDataType, OutputDataType, Residual, CustomLayers...>::DeleteModules() -{ - if (model == true) - { - for (LayerTypes& layer : network) - { - boost::apply_visitor(deleteVisitor, layer); - } - } + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, network[1])), network.front()); } template void Sequential< InputDataType, OutputDataType, Residual, CustomLayers...>::serialize( - Archive& ar, const unsigned int /* version */) + Archive& ar, const unsigned int version) { // If loading, delete the old layers. if (Archive::is_loading::value) @@ -209,6 +202,11 @@ void Sequential< ar & BOOST_SERIALIZATION_NVP(model); ar & BOOST_SERIALIZATION_NVP(network); + + if (version >= 1) + ar & BOOST_SERIALIZATION_NVP(ownsLayers); + else if (Archive::is_loading::value) + ownsLayers = !model; } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/softshrink.hpp b/src/mlpack/methods/ann/layer/softshrink.hpp new file mode 100644 index 0000000000..cc117bf361 --- /dev/null +++ b/src/mlpack/methods/ann/layer/softshrink.hpp @@ -0,0 +1,131 @@ +/** + * @file softshrink.hpp + * @author Lakshya Ojha + * + * The soft shrink function has threshold proportional to the noise level given + * by the user. + * The use of a Soft Shrink activation function provides adaptive denoising at + * various noise levels using a single CNN(Convolution Neural) without a + * requirement to train a unique CNN for each noise level. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_SOFTSHRINK_HPP +#define MLPACK_METHODS_ANN_LAYER_SOFTSHRINK_HPP + +#include + +namespace mlpack { +namespace ann /** Artifical Neural Network. */ { + +/** + * Soft Shrink operator is defined as, + * @f{eqnarray*}{ + * f(x) &=& \left\{ + * \begin{array}{lr} + * x - lambda & : x > lambda \\ + * x + lambda & : x < -lambda \\ + * 0 & : otherwise + * \end{array} \\ + * \right. + * f'(x) &=& \left\{ + * \begin{array}{lr} + * 1 & : x > lambda \\ + * 1 & : x < -lambda \\ + * 0 & : otherwise + * \end{array} + * \right. + * @f} + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class SoftShrink +{ + public: + /** + * Create Soft Shrink object using specified hyperparameter lambda. + * + * @param lambda The noise level of an image depends on settings of an + * imaging device. The settings can be used to select appropriate + * parameters for denoising methods. It is proportional to the noise + * level entered by the user. + * And it is calculated by multiplying the + * noise level sigma of the input(noisy image) and a + * coefficient 'a' which is one of the training parameters. + * Default value of lambda is 0.5. + */ + SoftShrink(const double lambda = 0.5); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the Soft Shrink function. + * @param output Resulting output activation + */ + template + void Forward(const InputType& input, OutputType& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards through f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation f(x). + * @param gy The backpropagated error. + * @param g The calculated gradient + */ + template + void Backward(const DataType& input, + DataType& gy, + DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the hyperparameter lambda. + double const& Lambda() const { return lambda; } + //! Modify the hyperparameter lambda. + double& Lambda() { return lambda; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored hyperparamater lambda. + double lambda; +}; // class SoftShrink + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "softshrink_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/softshrink_impl.hpp b/src/mlpack/methods/ann/layer/softshrink_impl.hpp new file mode 100644 index 0000000000..99876c89c8 --- /dev/null +++ b/src/mlpack/methods/ann/layer/softshrink_impl.hpp @@ -0,0 +1,61 @@ +/** + * @file softshrink.hpp + * @author Lakshya Ojha + * + * Implementation of Soft Shrink activation function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_SOFTSHRINK_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_SOFTSHRINK_IMPL_HPP + +// In case it hasn't yet been included +#include "softshrink.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +// This constructor is called for Soft Shrink activation function. +// lambda is a hyperparameter. +template +SoftShrink::SoftShrink(const double lambda) : + lambda(lambda) +{ + // Nothing to do here. +} + +template +template +void SoftShrink::Forward( + const InputType& input, OutputType& output) +{ + output = (input > lambda) % (input - lambda) + ( + input < -lambda) % (input + lambda); +} + +template +template +void SoftShrink::Backward( + const DataType& input, DataType& gy, DataType& g) +{ + DataType derivative; + derivative = (arma::ones(arma::size(input)) - (input == 0)); + g = gy % derivative; +} + +template +template +void SoftShrink::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(lambda); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/subview.hpp b/src/mlpack/methods/ann/layer/subview.hpp index f6dda59fb9..97ea9dd950 100644 --- a/src/mlpack/methods/ann/layer/subview.hpp +++ b/src/mlpack/methods/ann/layer/subview.hpp @@ -66,7 +66,7 @@ class Subview * @param output Resulting output activation. */ template - void Forward(InputType&& input, OutputType&& output) + void Forward(const InputType& input, OutputType& output) { size_t batchSize = input.n_cols / inSize; @@ -112,9 +112,9 @@ class Subview * @param g The calculated gradient. */ template - void Backward(arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g) + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy; } diff --git a/src/mlpack/methods/ann/layer/transposed_convolution.hpp b/src/mlpack/methods/ann/layer/transposed_convolution.hpp index 2ce2fb3b72..ef54fb9d24 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution.hpp @@ -144,7 +144,7 @@ class TransposedConvolution * @param output Resulting output activation. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -156,9 +156,9 @@ class TransposedConvolution * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -168,9 +168,9 @@ class TransposedConvolution * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } @@ -429,9 +429,6 @@ class TransposedConvolution //! Locally-stored transformed output parameter. arma::cube outputTemp; - //! Locally-stored transformed input parameter. - arma::cube inputTemp; - //! Locally-stored transformed padded input parameter. arma::cube inputPaddedTemp; diff --git a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp index c0ec126337..aac15f7595 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp @@ -207,10 +207,10 @@ void TransposedConvolution< GradientConvolutionRule, InputDataType, OutputDataType ->::Forward(const arma::Mat&& input, arma::Mat&& output) +>::Forward(const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; - inputTemp = arma::cube(const_cast&&>(input).memptr(), + arma::cube inputTemp(const_cast&>(input).memptr(), inputWidth, inputHeight, inSize * batchSize, false, false); if (strideWidth > 1 || strideHeight > 1) @@ -227,8 +227,8 @@ void TransposedConvolution< for (size_t i = 0; i < inputExpandedTemp.n_slices; ++i) { - paddingForward.Forward(std::move(inputExpandedTemp.slice(i)), - std::move(inputPaddedTemp.slice(i))); + paddingForward.Forward(inputExpandedTemp.slice(i), + inputPaddedTemp.slice(i)); } } else @@ -250,8 +250,7 @@ void TransposedConvolution< for (size_t i = 0; i < inputTemp.n_slices; ++i) { - paddingForward.Forward(std::move(inputTemp.slice(i)), - std::move(inputPaddedTemp.slice(i))); + paddingForward.Forward(inputTemp.slice(i), inputPaddedTemp.slice(i)); } } @@ -312,10 +311,10 @@ void TransposedConvolution< InputDataType, OutputDataType >::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { - arma::Cube mappedError(gy.memptr(), outputWidth, outputHeight, - outSize * batchSize, false, false); + arma::Cube mappedError(((arma::Mat&) gy).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); arma::Cube mappedErrorPadded; if (paddingBackward.PadWLeft() != 0 || paddingBackward.PadWRight() != 0 || paddingBackward.PadHTop() != 0 || paddingBackward.PadHBottom() != 0) @@ -327,13 +326,13 @@ void TransposedConvolution< for (size_t i = 0; i < mappedError.n_slices; ++i) { - paddingBackward.Forward(std::move(mappedError.slice(i)), - std::move(mappedErrorPadded.slice(i))); + paddingBackward.Forward(mappedError.slice(i), + mappedErrorPadded.slice(i)); } } - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); - gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, - inputTemp.n_cols, inputTemp.n_slices, false, false); + g.set_size(inputWidth * inputHeight * inSize, batchSize); + gTemp = arma::Cube(g.memptr(), inputWidth, inputHeight, inSize * + batchSize, false, false); gTemp.zeros(); @@ -382,12 +381,14 @@ void TransposedConvolution< InputDataType, OutputDataType >::Gradient( - const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - arma::Cube mappedError(error.memptr(), outputWidth, + arma::Cube mappedError(((arma::Mat&) error).memptr(), outputWidth, outputHeight, outSize * batchSize, false, false); + arma::cube inputTemp(const_cast&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); gradient.set_size(weights.n_elem, 1); gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, diff --git a/src/mlpack/methods/ann/layer/virtual_batch_norm.hpp b/src/mlpack/methods/ann/layer/virtual_batch_norm.hpp index 01ce3c5903..72b06e16b4 100644 --- a/src/mlpack/methods/ann/layer/virtual_batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/virtual_batch_norm.hpp @@ -76,7 +76,7 @@ class VirtualBatchNorm * @param output Resulting output activations. */ template - void Forward(const arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Backward pass through the layer. @@ -86,9 +86,9 @@ class VirtualBatchNorm * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& /* input */, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activations. @@ -98,9 +98,9 @@ class VirtualBatchNorm * @param gradient The calculated gradient. */ template - void Gradient(const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. OutputDataType const& Parameters() const { return weights; } diff --git a/src/mlpack/methods/ann/layer/virtual_batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/virtual_batch_norm_impl.hpp index b528f6ad98..250ef9392d 100644 --- a/src/mlpack/methods/ann/layer/virtual_batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/virtual_batch_norm_impl.hpp @@ -65,7 +65,7 @@ void VirtualBatchNorm::Reset() template template void VirtualBatchNorm::Forward( - const arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { inputParameter = input; arma::mat inputMean = arma::mean(input, 1); @@ -90,7 +90,7 @@ void VirtualBatchNorm::Forward( template template void VirtualBatchNorm::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); @@ -115,9 +115,9 @@ void VirtualBatchNorm::Backward( template template void VirtualBatchNorm::Gradient( - const arma::Mat&& /* input */, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient) { gradient.set_size(size + size, 1); diff --git a/src/mlpack/methods/ann/layer/vr_class_reward.hpp b/src/mlpack/methods/ann/layer/vr_class_reward.hpp index 7a6880f054..6c68673bd2 100644 --- a/src/mlpack/methods/ann/layer/vr_class_reward.hpp +++ b/src/mlpack/methods/ann/layer/vr_class_reward.hpp @@ -55,7 +55,7 @@ class VRClassReward * between 1 and the number of classes. */ template - double Forward(const InputType&& input, const TargetType&& target); + double Forward(const InputType& input, const TargetType& target); /** * Ordinary feed backward pass of a neural network. The negative log @@ -69,9 +69,9 @@ class VRClassReward * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const {return outputParameter; } diff --git a/src/mlpack/methods/ann/layer/vr_class_reward_impl.hpp b/src/mlpack/methods/ann/layer/vr_class_reward_impl.hpp index 679c314677..43a17d83e6 100644 --- a/src/mlpack/methods/ann/layer/vr_class_reward_impl.hpp +++ b/src/mlpack/methods/ann/layer/vr_class_reward_impl.hpp @@ -34,7 +34,7 @@ VRClassReward::VRClassReward( template template double VRClassReward::Forward( - const InputType&& input, const TargetType&& target) + const InputType& input, const TargetType& target) { double output = 0; for (size_t i = 0; i < input.n_cols - 1; ++i) @@ -66,9 +66,9 @@ double VRClassReward::Forward( template template void VRClassReward::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { output = arma::zeros(input.n_rows, input.n_cols); for (size_t i = 0; i < (input.n_cols - 1); ++i) diff --git a/src/mlpack/methods/ann/layer/weight_norm.hpp b/src/mlpack/methods/ann/layer/weight_norm.hpp index c4762c22d4..b07ee30e48 100644 --- a/src/mlpack/methods/ann/layer/weight_norm.hpp +++ b/src/mlpack/methods/ann/layer/weight_norm.hpp @@ -42,7 +42,8 @@ namespace ann /** Artificial Neural Network. */ { * Training of Deep Neural Networks}, * author = {Tim Salimans, Diederik P. Kingma}, * booktitle = {Neural Information Processing Systems 2016}, - * year = {2016} + * year = {2016}, + * url = {https://arxiv.org/abs/1602.07868}, * } * @endcode * @@ -85,7 +86,7 @@ class WeightNorm * @param output Resulting output activations. */ template - void Forward(arma::Mat&& input, arma::Mat&& output); + void Forward(const arma::Mat& input, arma::Mat& output); /** * Backward pass through the layer. This function calls the Backward() @@ -96,9 +97,9 @@ class WeightNorm * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& input, - arma::Mat&& gy, - arma::Mat&& g); + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta, input activations and the @@ -109,9 +110,9 @@ class WeightNorm * @param gradient The calculated gradient. */ template - void Gradient(arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient); + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the delta. OutputDataType const& Delta() const { return delta; } diff --git a/src/mlpack/methods/ann/layer/weight_norm_impl.hpp b/src/mlpack/methods/ann/layer/weight_norm_impl.hpp index d56b6156dc..1140106dfc 100644 --- a/src/mlpack/methods/ann/layer/weight_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/weight_norm_impl.hpp @@ -50,13 +50,12 @@ void WeightNorm::Reset() { // Set the weights of the inside layer to layerWeights. // This is done to set the non-bias terms correctly. - boost::apply_visitor(WeightSetVisitor(std::move(layerWeights), 0), - wrappedLayer); + boost::apply_visitor(WeightSetVisitor(layerWeights, 0), wrappedLayer); boost::apply_visitor(resetVisitor, wrappedLayer); - biasWeightSize = boost::apply_visitor(BiasSetVisitor(std::move(weights), - 0), wrappedLayer); + biasWeightSize = boost::apply_visitor(BiasSetVisitor(weights, 0), + wrappedLayer); vectorParameter = arma::mat(weights.memptr() + biasWeightSize, layerWeightSize - biasWeightSize, 1, false, false); @@ -69,15 +68,15 @@ template template void WeightNorm::Forward( - arma::Mat&& input, arma::Mat&& output) + const arma::Mat& input, arma::Mat& output) { // Initialize the non-bias weights of wrapped layer. const double normVectorParameter = arma::norm(vectorParameter, 2); layerWeights.rows(0, layerWeightSize - biasWeightSize - 1) = scalarParameter(0) * vectorParameter / normVectorParameter; - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, wrappedLayer))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, wrappedLayer)), wrappedLayer); output = boost::apply_visitor(outputParameterVisitor, wrappedLayer); @@ -87,11 +86,11 @@ template template void WeightNorm::Backward( - const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { - boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, wrappedLayer)), std::move(gy), std::move( - boost::apply_visitor(deltaVisitor, wrappedLayer))), wrappedLayer); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, wrappedLayer), gy, + boost::apply_visitor(deltaVisitor, wrappedLayer)), wrappedLayer); g = boost::apply_visitor(deltaVisitor, wrappedLayer); } @@ -100,15 +99,14 @@ template template void WeightNorm::Gradient( - arma::Mat&& input, - arma::Mat&& error, - arma::Mat&& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { ResetGradients(layerGradients); // Calculate the gradients of the wrapped layer. - boost::apply_visitor(GradientVisitor(std::move(input), - std::move(error)), wrappedLayer); + boost::apply_visitor(GradientVisitor(input, error), wrappedLayer); // Store the norm of vector parameter temporarily. const double normVectorParameter = arma::norm(vectorParameter, 2); @@ -137,8 +135,7 @@ template::ResetGradients( arma::mat& gradient) { - boost::apply_visitor(GradientSetVisitor(std::move(gradient), 0), - wrappedLayer); + boost::apply_visitor(GradientSetVisitor(gradient, 0), wrappedLayer); } template { } - /* + /** * Return the name of the given layer of type AtrousConvolution as a string. * * @param Given layer of type AtrousConvolution. @@ -41,7 +41,7 @@ class LayerNameVisitor : public boost::static_visitor return "atrousconvolution"; } - /* + /** * Return the name of the given layer of type AlphaDropout as a string. * * @param Given layer of type AlphaDropout. @@ -52,7 +52,7 @@ class LayerNameVisitor : public boost::static_visitor return "alphadropout"; } - /* + /** * Return the name of the given layer of type BatchNorm as a string. * * @param Given layer of type BatchNorm. @@ -63,7 +63,7 @@ class LayerNameVisitor : public boost::static_visitor return "batchnorm"; } - /* + /** * Return the name of the given layer of type Constant as a string. * * @param Given layer of type Constant. @@ -74,7 +74,7 @@ class LayerNameVisitor : public boost::static_visitor return "constant"; } - /* + /** * Return the name of the given layer of type Convolution as a string. * * @param Given layer of type Convolution. @@ -85,7 +85,7 @@ class LayerNameVisitor : public boost::static_visitor return "convolution"; } - /* + /** * Return the name of the given layer of type DropConnect as a string. * * @param Given layer of type DropConnect. @@ -96,7 +96,7 @@ class LayerNameVisitor : public boost::static_visitor return "dropconnect"; } - /* + /** * Return the name of the given layer of type Dropout as a string. * * @param Given layer of type Dropout. @@ -107,7 +107,7 @@ class LayerNameVisitor : public boost::static_visitor return "dropout"; } - /* + /** * Return the name of the given layer of type FlexibleReLU as a string. * * @param Given layer of type FlexibleReLU. @@ -118,7 +118,7 @@ class LayerNameVisitor : public boost::static_visitor return "flexiblerelu"; } - /* + /** * Return the name of the given layer of type LayerNorm as a string. * * @param Given layer of type LayerNorm. @@ -129,7 +129,7 @@ class LayerNameVisitor : public boost::static_visitor return "layernorm"; } - /* + /** * Return the name of the given layer of type Linear as a string. * * @param Given layer of type Linear. @@ -140,7 +140,7 @@ class LayerNameVisitor : public boost::static_visitor return "linear"; } - /* + /** * Return the name of the given layer of type LinearNoBias as a string. * * @param Given layer of type LinearNoBias. @@ -151,7 +151,7 @@ class LayerNameVisitor : public boost::static_visitor return "linearnobias"; } - /* + /** * Return the name of the given layer of type MaxPooling as a string. * * @param Given layer of type MaxPooling. @@ -162,7 +162,7 @@ class LayerNameVisitor : public boost::static_visitor return "maxpooling"; } - /* + /** * Return the name of the given layer of type MeanPooling as a string. * * @param Given layer of type MeanPooling. @@ -173,7 +173,7 @@ class LayerNameVisitor : public boost::static_visitor return "meanpooling"; } - /* + /** * Return the name of the given layer of type MultiplyConstant as a string. * * @param Given layer of type MultiplyConstant. @@ -184,7 +184,7 @@ class LayerNameVisitor : public boost::static_visitor return "multiplyconstant"; } - /* + /** * Return the name of the given layer of type ReLULayer as a string. * * @param Given layer of type ReLULayer. @@ -195,7 +195,7 @@ class LayerNameVisitor : public boost::static_visitor return "relu"; } - /* + /** * Return the name of the given layer of type TransposedConvolution as a * string. * @@ -207,7 +207,7 @@ class LayerNameVisitor : public boost::static_visitor return "transposedconvolution"; } - /* + /** * Return the name of the given layer of type IdentityLayer as a string. * * @param Given layer of type IdentityLayer. @@ -218,7 +218,7 @@ class LayerNameVisitor : public boost::static_visitor return "identity"; } - /* + /** * Return the name of the given layer of type TanHLayer as a string. * * @param Given layer of type TanHLayer. @@ -229,7 +229,7 @@ class LayerNameVisitor : public boost::static_visitor return "tanh"; } - /* + /** * Return the name of the given layer of type ELU as a string. * * @param Given layer of type ELU. @@ -240,7 +240,7 @@ class LayerNameVisitor : public boost::static_visitor return "elu"; } - /* + /** * Return the name of the given layer of type HardTanH as a string. * * @param Given layer of type HardTanH. @@ -251,7 +251,7 @@ class LayerNameVisitor : public boost::static_visitor return "hardtanh"; } - /* + /** * Return the name of the given layer of type LeakyReLU as a string. * * @param Given layer of type LeakyReLU. @@ -262,7 +262,7 @@ class LayerNameVisitor : public boost::static_visitor return "leakyrelu"; } - /* + /** * Return the name of the given layer of type PReLU as a string. * * @param Given layer of type PReLU. @@ -273,7 +273,7 @@ class LayerNameVisitor : public boost::static_visitor return "prelu"; } - /* + /** * Return the name of the given layer of type SigmoidLayer as a string. * * @param Given layer of type SigmoidLayer. @@ -284,7 +284,7 @@ class LayerNameVisitor : public boost::static_visitor return "sigmoid"; } - /* + /** * Return the name of the given layer of type LogSoftMax as a string. * * @param Given layer of type LogSoftMax. @@ -306,7 +306,7 @@ class LayerNameVisitor : public boost::static_visitor return "lstm"; } - /* + /** * Return the name of the given layer of type CReLU as a string. * * @param Given layer of type CReLU. @@ -317,7 +317,7 @@ class LayerNameVisitor : public boost::static_visitor return "crelu"; } - /* + /** * Return the name of the given layer of type Highway as a string. * * @param Given layer of type Highway. @@ -328,7 +328,7 @@ class LayerNameVisitor : public boost::static_visitor return "highway"; } - /* + /** * Return the name of the given layer of type GRU as a string. * * @param Given layer of type GRU. @@ -339,7 +339,7 @@ class LayerNameVisitor : public boost::static_visitor return "gru"; } - /* + /** * Return the name of the given layer of type Glimpse as a string. * * @param Given layer of type Glimpse. @@ -350,7 +350,7 @@ class LayerNameVisitor : public boost::static_visitor return "glimpse"; } - /* + /** * Return the name of the given layer of type FastLSTM as a string. * * @param Given layer of type FastLSTM. @@ -361,7 +361,7 @@ class LayerNameVisitor : public boost::static_visitor return "fastlstm"; } - /* + /** * Return the name of the given layer of type WeightNorm as a string. * * @param Given layer of type WeightNorm. @@ -372,7 +372,7 @@ class LayerNameVisitor : public boost::static_visitor return "weightnorm"; } - /* + /** * Return the name of the layer of specified type as a string. * * @param Given layer of any type. diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index a1bdcb8fd8..c20856a214 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -3,12 +3,18 @@ set(SOURCES cross_entropy_error.hpp cross_entropy_error_impl.hpp + cosine_embedding_loss.hpp + cosine_embedding_loss_impl.hpp dice_loss.hpp dice_loss_impl.hpp earth_mover_distance.hpp earth_mover_distance_impl.hpp + huber_loss.hpp + huber_loss_impl.hpp kl_divergence.hpp kl_divergence_impl.hpp + margin_ranking_loss.hpp + margin_ranking_loss_impl.hpp mean_bias_error.hpp mean_bias_error_impl.hpp mean_squared_error.hpp @@ -23,6 +29,8 @@ set(SOURCES reconstruction_loss_impl.hpp sigmoid_cross_entropy_error.hpp sigmoid_cross_entropy_error_impl.hpp + hinge_embedding_loss.hpp + hinge_embedding_loss_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp new file mode 100644 index 0000000000..044fbbdd0e --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -0,0 +1,141 @@ +/** + * @file cosine_embedding_loss.hpp + * @author Kartik Dutt + * + * Definition of the Cosine Embedding loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_COSINE_EMBEDDING_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_COSINE_EMBEDDING_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Cosine Embedding Loss function is used for measuring whether two inputs are + * similar or dissimilar, using the cosine distance, and is typically used + * for learning nonlinear embeddings or semi-supervised learning. + * + * @f{eqnarray*}{ + * f(x) = 1 - cos(x1, x2) , for y = 1 + * f(x) = max(0, cos(x1, x2) - margin) , for y = -1 + * @f} + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class CosineEmbeddingLoss +{ + public: + /** + * Create the CosineEmbeddingLoss object. + * + * @param margin Increases cosine distance in case of dissimilarity. + * Refer definition of cosine-embedding-loss above. + * @param similarity Determines whether to use similarity or dissimilarity for + * comparision. + * @param takeMean Boolean variable to specify whether to take mean or not. + * Specifies reduction method i.e. sum or mean corresponding + * to 0 and 1 respectively. Default value = 0. + */ + CosineEmbeddingLoss(const double margin = 0.0, + const bool similarity = true, + const bool takeMean = false); + + /** + * Ordinary feed forward pass of a neural network. + * + * @param input Input data used for evaluating the specified function. + * @param target The target vector. + */ + template + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); + + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated input activation. + * @param target The target vector. + * @param output The calculated error. + */ + template + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); + + //! Get the input parameter. + InputDataType& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the value of takeMean. + bool TakeMean() const { return takeMean; } + //! Modify the value of takeMean. + bool& TakeMean() { return takeMean; } + + //! Get the value of margin. + double Margin() const { return margin; } + //! Modify the value of takeMean. + double& Margin() { return margin; } + + //! Get the value of similarity hyperparameter. + bool Similarity() const { return similarity; } + //! Modify the value of takeMean. + bool& Similarity() { return similarity; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored value of similarity hyper-parameter. + bool similarity; + + //! Locally-stored value of margin hyper-parameter. + double margin; + + //! Locally-stored value of takeMean hyper-parameter. + bool takeMean; +}; // class CosineEmbeddingLoss + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "cosine_embedding_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp new file mode 100644 index 0000000000..b708567b28 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp @@ -0,0 +1,120 @@ +/** + * @file cosine_embedding_loss_impl.hpp + * @author Kartik Dutt + * + * Implementation of the Cosine Embedding loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_COSINE_EMBEDDING_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_COSINE_EMBEDDING_IMPL_HPP + +// In case it hasn't yet been included. +#include "cosine_embedding_loss.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +CosineEmbeddingLoss::CosineEmbeddingLoss( + const double margin, const bool similarity, const bool takeMean): + margin(margin), similarity(similarity), takeMean(takeMean) +{ + // Nothing to do here. +} + +template +template +typename InputType::elem_type +CosineEmbeddingLoss::Forward( + const InputType& input, + const TargetType& target) +{ + typedef typename InputType::elem_type ElemType; + + const size_t cols = input.n_cols; + const size_t batchSize = input.n_elem / cols; + if (arma::size(input) != arma::size(target)) + Log::Fatal << "Input Tensors must have same dimensions." << std::endl; + + arma::colvec inputTemp1 = arma::vectorise(input); + arma::colvec inputTemp2 = arma::vectorise(target); + ElemType loss = 0.0; + + for (size_t i = 0; i < inputTemp1.n_elem; i += cols) + { + const ElemType cosDist = kernel::CosineDistance::Evaluate( + inputTemp1(arma::span(i, i + cols - 1)), inputTemp2(arma::span(i, + i + cols - 1))); + if (similarity) + loss += 1 - cosDist; + else + { + const ElemType currentLoss = cosDist - margin; + loss += currentLoss > 0 ? currentLoss : 0; + } + } + + if (takeMean) + loss = (ElemType) loss / batchSize; + + return loss; +} + +template +template +void CosineEmbeddingLoss::Backward( + const InputType& input, + const TargetType& target, + OutputType& output) +{ + typedef typename InputType::elem_type ElemType; + + const size_t cols = input.n_cols; + const size_t batchSize = input.n_elem / cols; + if (arma::size(input) != arma::size(target)) + Log::Fatal << "Input Tensors must have same dimensions." << std::endl; + + arma::colvec inputTemp1 = arma::vectorise(input); + arma::colvec inputTemp2 = arma::vectorise(target); + output.set_size(arma::size(inputTemp1)); + + arma::colvec outputTemp(output.memptr(), inputTemp1.n_elem, + false, false); + for (size_t i = 0; i < inputTemp1.n_elem; i += cols) + { + const ElemType cosDist = kernel::CosineDistance::Evaluate(inputTemp1( + arma::span(i, i + cols -1)), inputTemp2(arma::span(i, i + cols -1))); + + if (cosDist < margin && !similarity) + outputTemp(arma::span(i, i + cols - 1)).zeros(); + else + { + const int multiplier = similarity ? 1 : -1; + outputTemp(arma::span(i, i + cols -1)) = -1 * multiplier * + (arma::normalise(inputTemp2(arma::span(i, i + cols - 1))) - + cosDist * arma::normalise(inputTemp1(arma::span(i, i + cols - + 1)))) / std::sqrt(arma::accu(arma::pow(inputTemp1(arma::span(i, i + + cols - 1)), 2))); + } + } +} + +template +template +void CosineEmbeddingLoss::serialize( + Archive& ar , + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(margin); + ar & BOOST_SERIALIZATION_NVP(similarity); + ar & BOOST_SERIALIZATION_NVP(takeMean); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp index 87d350eeb7..1811f99998 100644 --- a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp @@ -49,7 +49,8 @@ class CrossEntropyError * @param target The target vector. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -59,9 +60,9 @@ class CrossEntropyError * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/cross_entropy_error_impl.hpp index 1edc6df5ee..7d0d80357b 100644 --- a/src/mlpack/methods/ann/loss_functions/cross_entropy_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/cross_entropy_error_impl.hpp @@ -27,8 +27,10 @@ CrossEntropyError::CrossEntropyError( template template -double CrossEntropyError::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type +CrossEntropyError::Forward( + const InputType& input, + const TargetType& target) { return -arma::accu(target % arma::log(input + eps) + (1. - target) % arma::log(1. - input + eps)); @@ -37,9 +39,9 @@ double CrossEntropyError::Forward( template template void CrossEntropyError::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { output = (1. - target) / (1. - input + eps) - target / (input + eps); } diff --git a/src/mlpack/methods/ann/loss_functions/dice_loss.hpp b/src/mlpack/methods/ann/loss_functions/dice_loss.hpp index 4b10db6c5b..42551388aa 100644 --- a/src/mlpack/methods/ann/loss_functions/dice_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/dice_loss.hpp @@ -62,7 +62,8 @@ class DiceLoss * @param target The target vector. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -72,9 +73,9 @@ class DiceLoss * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp index 77413de5bd..904d699c21 100644 --- a/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp @@ -27,8 +27,9 @@ DiceLoss::DiceLoss( template template -double DiceLoss::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type DiceLoss::Forward( + const InputType& input, + const TargetType& target) { return 1 - ((2 * arma::accu(target % input) + smooth) / (arma::accu(target % target) + arma::accu( @@ -38,9 +39,9 @@ double DiceLoss::Forward( template template void DiceLoss::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { output = -2 * (target * (arma::accu(input % input) + arma::accu(target % target) + smooth) - input * diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp index 14e5daa8d5..71c0aa8e4b 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp @@ -45,7 +45,8 @@ class EarthMoverDistance * @param target The target vector. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -55,9 +56,9 @@ class EarthMoverDistance * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp index b7db1786a4..44d5e24c4e 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp @@ -26,8 +26,10 @@ EarthMoverDistance::EarthMoverDistance() template template -double EarthMoverDistance::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type +EarthMoverDistance::Forward( + const InputType& input, + const TargetType& target) { return -arma::accu(target % input); } @@ -35,9 +37,9 @@ double EarthMoverDistance::Forward( template template void EarthMoverDistance::Backward( - const InputType&& /* input */, - const TargetType&& target, - OutputType&& output) + const InputType& /* input */, + const TargetType& target, + OutputType& output) { output = -target; } diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp new file mode 100644 index 0000000000..3e3eac19ec --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp @@ -0,0 +1,88 @@ +/** + * @file hinge_embedding_loss.hpp + * @author Lakshya Ojha + * + * Definition of the Hinge Embedding Loss Function. + * The Hinge Embedding loss function is often used to improve performance + * in semi-supervised learning or to learn nonlinear embeddings. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_HINGE_EMBEDDING_LOSS_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_HINGE_EMBEDDING_LOSS_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The Hinge Embedding loss function is often used to compute the loss + * between y_true and y_pred. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class HingeEmbeddingLoss +{ + public: + /** + * Create the Hinge Embedding object. + */ + HingeEmbeddingLoss(); + + /** + * Computes the Hinge Embedding loss function. + * + * @param input Input data used for evaluating the specified function. + * @param target Target data to compare with. + */ + template + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); + + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated input activation. + * @param target The target vector. + * @param output The calculated error. + */ + template + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + /** + * Serialize the loss function. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class HingeEmbeddingLoss + +} // namespace ann +} // namespace mlpack + +// include implementation +#include "hinge_embedding_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp new file mode 100644 index 0000000000..f3f420b3ca --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp @@ -0,0 +1,62 @@ +/** + * @file hinge_embedding_loss_impl.hpp + * @author Lakshya Ojha + * + * Implementation of the Hinge Embedding loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_HINGE_EMBEDDING_LOSS_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_HINGE_EMBEDDING_LOSS_IMPL_HPP + +// In case it hasn't yet been included. +#include "hinge_embedding_loss.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +HingeEmbeddingLoss::HingeEmbeddingLoss() +{ + // Nothing to do here. +} + +template +template +typename InputType::elem_type +HingeEmbeddingLoss::Forward( + const InputType& input, + const TargetType& target) +{ + TargetType temp = target - (target == 0); + return (arma::accu(arma::max(1-input % temp, 0.))) / target.n_elem; +} + +template +template +void HingeEmbeddingLoss::Backward( + const InputType& input, + const TargetType& target, + OutputType& output) +{ + TargetType temp = target - (target == 0); + output = (input < 1 / temp) % -temp; +} + +template +template +void HingeEmbeddingLoss::serialize( + Archive& /* ar */, + const unsigned int /* version */) +{ + // Nothing to do here. +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp new file mode 100644 index 0000000000..60e8d96b10 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp @@ -0,0 +1,108 @@ +/** + * @file huber_loss.hpp + * @author Mrityunjay Tripathi + * + * Definition of the Huber loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_HUBER_LOSS_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_HUBER_LOSS_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The Huber loss is a loss function used in robust regression, + * that is less sensitive to outliers in data than the squared error loss. + * This function is quadratic for small values of \f$ y - f(x) \f$, + * and linear for large values, with equal values and slopes of the different + * sections at the two points where \f$ |y - f(x)| = delta \f$. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class HuberLoss +{ + public: + /** + * Create the HuberLoss object. + * + * @param delta The threshold value upto which squared error is followed and + * after which absolute error is considered. + * @param mean If true then mean loss is computed otherwise sum. + */ + HuberLoss(const double delta = 1.0, const bool mean = true); + + /** + * Computes the Huber Loss function. + * + * @param input Input data used for evaluating the specified function. + * @param target The target vector. + */ + template + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); + + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated input activation. + * @param target The target vector. + * @param output The calculated error. + */ + template + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the value of delta. + double Delta() const { return delta; } + //! Set the value of delta. + double& Delta() { return delta; } + + //! Get the value of reduction type. + bool Mean() const { return mean; } + //! Set the value of reduction type. + bool& Mean() { return mean; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Hyperparameter `delta` defines the point upto which MSE is considered. + double delta; + + //! Reduction type. If true, performs mean of loss else sum. + bool mean; +}; // class HuberLoss + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "huber_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp new file mode 100644 index 0000000000..c9c5ee9145 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp @@ -0,0 +1,81 @@ +/** + * @file huber_loss_impl.hpp + * @author Mrityunjay Tripathi + * + * Implementation of the Huber loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_HUBER_LOSS_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_HUBER_LOSS_IMPL_HPP + +// In case it hasn't yet been included. +#include "huber_loss.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +HuberLoss::HuberLoss( + const double delta, + const bool mean): + delta(delta), + mean(mean) +{ + // Nothing to do here. +} + +template +template +typename InputType::elem_type +HuberLoss::Forward(const InputType& input, + const TargetType& target) +{ + typedef typename InputType::elem_type ElemType; + ElemType loss = 0; + for (size_t i = 0; i < input.n_elem; ++i) + { + const ElemType absError = std::abs(target[i] - input[i]); + loss += absError > delta + ? delta * (absError - 0.5 * delta) : 0.5 * std::pow(absError, 2); + } + return mean ? loss / input.n_elem : loss; +} + +template +template +void HuberLoss::Backward( + const InputType& input, + const TargetType& target, + OutputType& output) +{ + typedef typename InputType::elem_type ElemType; + + output.set_size(size(input)); + for (size_t i = 0; i < output.n_elem; ++i) + { + const ElemType absError = std::abs(target[i] - input[i]); + output[i] = absError > delta + ? - delta * (target[i] - input[i]) / absError : input[i] - target[i]; + if (mean) + output[i] /= output.n_elem; + } +} + +template +template +void HuberLoss::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(delta); + ar & BOOST_SERIALIZATION_NVP(mean); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index 87f32de6f9..4640dce43e 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -60,7 +60,8 @@ class KLDivergence * @param target Target data to compare with. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -70,9 +71,9 @@ class KLDivergence * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp index 590878ed3b..bc9b44ea09 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp @@ -28,8 +28,9 @@ KLDivergence::KLDivergence(const bool takeMean) : template template -double KLDivergence::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type +KLDivergence::Forward(const InputType& input, + const TargetType& target) { if (takeMean) { @@ -45,9 +46,9 @@ double KLDivergence::Forward( template template void KLDivergence::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { if (takeMean) { @@ -62,10 +63,10 @@ void KLDivergence::Backward( template template void KLDivergence::serialize( - Archive& /* ar */, + Archive& ar, const unsigned int /* version */) { - // Nothing to do here. + ar & BOOST_SERIALIZATION_NVP(takeMean); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp index 650c87a94d..2fc859c5c7 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp @@ -20,7 +20,7 @@ namespace ann /** Artificial Neural Network. */ { /** * The Log-Hyperbolic-Cosine loss function is often used to improve - * variational auto encoder. This function is the log of hyperbolic + * variational auto encoder. This function is the log of hyperbolic * cosine of difference between true values and predicted values. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, @@ -55,7 +55,8 @@ class LogCoshLoss * @param target Target data to compare with. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -65,9 +66,9 @@ class LogCoshLoss * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } @@ -75,9 +76,9 @@ class LogCoshLoss OutputDataType& OutputParameter() { return outputParameter; } //! Get the value of hyperparameter a. - bool A() const { return a; } + double A() const { return a; } //! Modify the value of hyperparameter a. - bool& A() { return a; } + double& A() { return a; } /** * Serialize the loss function. diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp index 9dc427ded3..1fd13c922f 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp @@ -28,8 +28,9 @@ LogCoshLoss::LogCoshLoss(const double a) : template template -double LogCoshLoss::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type +LogCoshLoss::Forward(const InputType& input, + const TargetType& target) { return arma::accu(arma::log(arma::cosh(a * (target - input)))) / a; } @@ -37,9 +38,9 @@ double LogCoshLoss::Forward( template template void LogCoshLoss::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { output = arma::tanh(a * (target - input)); } @@ -47,10 +48,10 @@ void LogCoshLoss::Backward( template template void LogCoshLoss::serialize( - Archive& /* ar */, + Archive& ar, const unsigned int /* version */) { - // Nothing to do here. + ar & BOOST_SERIALIZATION_NVP(a); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp new file mode 100644 index 0000000000..e8798dd62b --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -0,0 +1,102 @@ +/** + * @file margin_ranking_loss.hpp + * @author Andrei Mihalea + * + * Definition of the Margin Ranking Loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_ANN_LOSS_FUNCTION_MARGIN_RANKING_LOSS_HPP +#define MLPACK_ANN_LOSS_FUNCTION_MARGIN_RANKING_LOSS_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Margin ranking loss measures the loss given inputs and a label vector with + * values of 1 or -1. If the label is 1 then the first input should be ranked + * higher than the second input at a distance larger than a margin, and vice- + * versa if the label is -1. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MarginRankingLoss +{ + public: + /** + * Create the MarginRankingLoss object with Hyperparameter margin. + * Hyperparameter margin defines a minimum distance between correctly ranked + * samples. + */ + MarginRankingLoss(const double margin = 1.0); + + /** + * Computes the Margin Ranking Loss function. + * + * @param input Concatenation of the two inputs for evaluating the specified + * function. + * @param target The label vector which contains values of -1 or 1. + */ + template + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); + + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated concatenated input activation. + * @param target The label vector which contains -1 or 1 values. + * @param output The calculated error. + */ + template < + typename InputType, + typename TargetType, + typename OutputType + > + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the margin parameter. + double Margin() const { return margin; } + //! Modify the margin parameter. + double& Margin() { return margin; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! The margin value used in calculating Margin Ranking Loss. + double margin; +}; // class MarginRankingLoss + +} // namespace ann +} // namespace mlpack + +// include implementation. +#include "margin_ranking_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp new file mode 100644 index 0000000000..59649cbb3a --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -0,0 +1,74 @@ +/** + * @file margin_ranking_loss_impl.hpp + * @author Andrei Mihalea + * + * Implementation of the Margin Ranking Loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_MARGIN_IMPL_LOSS_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_MARGIN_IMPL_LOSS_HPP + +// In case it hasn't been included. +#include "margin_ranking_loss.hpp" + +namespace mlpack { +namespace ann /** Artifical Neural Network. */ { + +template +MarginRankingLoss::MarginRankingLoss( + const double margin) : margin(margin) +{ + // Nothing to do here. +} + +template +template +typename InputType::elem_type +MarginRankingLoss::Forward( + const InputType& input, + const TargetType& target) +{ + const int inputRows = input.n_rows; + const InputType& input1 = input.rows(0, inputRows / 2 - 1); + const InputType& input2 = input.rows(inputRows / 2, inputRows - 1); + return arma::accu(arma::max(arma::zeros(size(target)), + -target % (input1 - input2) + margin)) / target.n_cols; +} + +template +template < + typename InputType, + typename TargetType, + typename OutputType +> +void MarginRankingLoss::Backward( + const InputType& input, + const TargetType& target, + OutputType& output) +{ + const int inputRows = input.n_rows; + const InputType& input1 = input.rows(0, inputRows / 2 - 1); + const InputType& input2 = input.rows(inputRows / 2, inputRows - 1); + output = -target % (input1 - input2) + margin; + output.elem(arma::find(output >= 0)).ones(); + output.elem(arma::find(output < 0)).zeros(); + output = (input2 - input1) % output / target.n_cols; +} + +template +template +void MarginRankingLoss::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(margin); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp index 1ec50e67b5..a238ac50cb 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp @@ -45,7 +45,8 @@ class MeanBiasError * @param target The target vector. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -55,9 +56,9 @@ class MeanBiasError * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp index 4a7a2114d2..9d014585f4 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp @@ -27,8 +27,9 @@ MeanBiasError::MeanBiasError() template template -double MeanBiasError::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type +MeanBiasError::Forward(const InputType& input, + const TargetType& target) { return arma::accu(target - input) / target.n_cols; } @@ -36,9 +37,9 @@ double MeanBiasError::Forward( template template void MeanBiasError::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& /* target */, + OutputType& output) { output.set_size(arma::size(input)); output.fill(-1.0); diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 6ab98d1791..0315c10b5c 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -46,7 +46,8 @@ class MeanSquaredError * @param target The target vector. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -56,9 +57,9 @@ class MeanSquaredError * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp index 82c9cc3bd9..81cf2281cf 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp @@ -26,8 +26,10 @@ MeanSquaredError::MeanSquaredError() template template -double MeanSquaredError::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type +MeanSquaredError::Forward( + const InputType& input, + const TargetType& target) { return arma::accu(arma::square(input - target)) / target.n_cols; } @@ -35,9 +37,9 @@ double MeanSquaredError::Forward( template template void MeanSquaredError::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { output = 2 * (input - target) / target.n_cols; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp index d6cf0b30f9..49b94d1d4f 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp @@ -45,7 +45,8 @@ class MeanSquaredLogarithmicError * @param target The target vector. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -55,9 +56,9 @@ class MeanSquaredLogarithmicError * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp index 61e21c159e..92ead103a7 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp @@ -27,8 +27,10 @@ MeanSquaredLogarithmicError template template -double MeanSquaredLogarithmicError::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type +MeanSquaredLogarithmicError::Forward( + const InputType& input, + const TargetType& target) { return arma::accu(arma::square(arma::log(1. + target) - arma::log(1. + input))) / target.n_cols; @@ -37,9 +39,9 @@ double MeanSquaredLogarithmicError::Forward( template template void MeanSquaredLogarithmicError::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { output = 2 * (arma::log(1. + input) - arma::log(1. + target)) / ((1. + input) * target.n_cols); diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index fdc5433eaf..6c28321cb9 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -48,7 +48,8 @@ class NegativeLogLikelihood * between 1 and the number of classes. */ template - double Forward(const InputType&& input, TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. The negative log @@ -62,9 +63,9 @@ class NegativeLogLikelihood * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the input parameter. InputDataType& InputParameter() const { return inputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp index 922546753b..1eb47280c5 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -26,10 +26,13 @@ NegativeLogLikelihood::NegativeLogLikelihood() template template -double NegativeLogLikelihood::Forward( - const InputType&& input, TargetType&& target) +typename InputType::elem_type +NegativeLogLikelihood::Forward( + const InputType& input, + const TargetType& target) { - double output = 0; + typedef typename InputType::elem_type ElemType; + ElemType output = 0; for (size_t i = 0; i < input.n_cols; ++i) { size_t currentTarget = target(i) - 1; @@ -45,9 +48,9 @@ double NegativeLogLikelihood::Forward( template template void NegativeLogLikelihood::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { output = arma::zeros(input.n_rows, input.n_cols); for (size_t i = 0; i < input.n_cols; ++i) diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index 71cf33cd3c..7d7c8e7da6 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -49,7 +49,8 @@ class ReconstructionLoss * @param target The target matrix. */ template - double Forward(const InputType&& input, const TargetType&& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -59,9 +60,9 @@ class ReconstructionLoss * @param output The calculated error. */ template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp index 73fd57f64d..b47d5bfcdb 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -30,21 +30,22 @@ ReconstructionLoss< template template -double ReconstructionLoss::Forward( - const InputType&& input, const TargetType&& target) +typename InputType::elem_type +ReconstructionLoss::Forward( + const InputType& input, const TargetType& target) { - dist = DistType(std::move(input)); - return -dist.LogProbability(std::move(target)); + dist = DistType(input); + return -dist.LogProbability(target); } template template void ReconstructionLoss::Backward( - const InputType&& /* input */, - const TargetType&& target, - OutputType&& output) + const InputType& /* input */, + const TargetType& target, + OutputType& output) { - dist.LogProbBackward(std::move(target), std::move(output)); + dist.LogProbBackward(target, output); output *= -1; } diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index 6021d36e2d..0d70d2d29d 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -64,8 +64,8 @@ class SigmoidCrossEntropyError * @param target The target vector. */ template - inline double Forward(const InputType&& input, - const TargetType&& target); + inline typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. * @@ -74,9 +74,9 @@ class SigmoidCrossEntropyError * @param output The calculated error. */ template - inline void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + inline void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp index 853729a0b5..e5cf69188c 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp @@ -29,10 +29,13 @@ SigmoidCrossEntropyError template template -inline double SigmoidCrossEntropyError::Forward( - const InputType&& input, const TargetType&& target) +inline typename InputType::elem_type +SigmoidCrossEntropyError::Forward( + const InputType& input, + const TargetType& target) { - double maximum = 0; + typedef typename InputType::elem_type ElemType; + ElemType maximum = 0; for (size_t i = 0; i < input.n_elem; ++i) { maximum += std::max(input[i], 0.0) + @@ -45,9 +48,9 @@ inline double SigmoidCrossEntropyError::Forward( template template inline void SigmoidCrossEntropyError::Backward( - const InputType&& input, - const TargetType&& target, - OutputType&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { output = 1.0 / (1.0 + arma::exp(-input)) - target; } diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp index c94d647518..11cb8a8001 100644 --- a/src/mlpack/methods/ann/rnn.hpp +++ b/src/mlpack/methods/ann/rnn.hpp @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -72,6 +73,35 @@ class RNN //! Destructor to release allocated memory. ~RNN(); + /** + * Check if the optimizer has MaxIterations() parameter, if it does + * then check if it's value is less than the number of datapoints + * in the dataset. + * + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. + */ + template + typename std::enable_if< + HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; + + /** + * Check if the optimizer has MaxIterations() parameter, if it + * doesn't then simply return from the function. + * + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. + */ + template + typename std::enable_if< + !HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; + /** * Train the recurrent neural network on the given input data using the given * optimizer. @@ -295,7 +325,8 @@ class RNN * * @param input Data sequence to compute probabilities for. */ - void Forward(arma::mat&& input); + template + void Forward(const InputType& input); /** * Reset the state of RNN cells in the network for new input sequence. @@ -313,7 +344,7 @@ class RNN * layer defined optimizer. */ template - void Gradient(InputType&& input); + void Gradient(const InputType& input); /** * Reset the module status by setting the current deterministic parameter diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp index 054d38923e..688f76d654 100644 --- a/src/mlpack/methods/ann/rnn_impl.hpp +++ b/src/mlpack/methods/ann/rnn_impl.hpp @@ -61,6 +61,40 @@ RNN::~RNN() } } +template +template +typename std::enable_if< + HasMaxIterations + ::value, void>::type +RNN:: +WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const +{ + if (optimizer.MaxIterations() < samples && + optimizer.MaxIterations() != 0) + { + Log::Warn << "The optimizer's maximum number of iterations " + << "is less than the size of the dataset; the " + << "optimizer will not pass over the entire " + << "dataset. To fix this, modify the maximum " + << "number of iterations to be at least equal " + << "to the number of points of your dataset " + << "(" << samples << ")." << std::endl; + } +} + +template +template +typename std::enable_if< + !HasMaxIterations + ::value, void>::type +RNN:: +WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const +{ + return; +} + template template @@ -83,6 +117,8 @@ double RNN::Train( ResetParameters(); } + WarnMessageMaxIterations(optimizer, this->predictors.n_cols); + // Train the model. Timer::Start("rnn_optimization"); const double out = optimizer.Optimize(*this, parameter, callbacks...); @@ -127,6 +163,8 @@ double RNN::Train( OptimizerType optimizer; + WarnMessageMaxIterations(optimizer, this->predictors.n_cols); + // Train the model. Timer::Start("rnn_optimization"); const double out = optimizer.Optimize(*this, parameter, callbacks...); @@ -158,8 +196,8 @@ void RNN::Predict( const size_t effectiveBatchSize = std::min(batchSize, size_t(predictors.n_cols)); - Forward(std::move(arma::mat(predictors.slice(0).colptr(0), - predictors.n_rows, effectiveBatchSize, false, true))); + Forward(arma::mat(predictors.slice(0).colptr(0), predictors.n_rows, + effectiveBatchSize, false, true)); arma::mat resultsTemp = boost::apply_visitor(outputParameterVisitor, network.back()); @@ -175,8 +213,8 @@ void RNN::Predict( size_t(predictors.n_cols - begin)); for (size_t seqNum = !begin; seqNum < rho; ++seqNum) { - Forward(std::move(arma::mat(predictors.slice(seqNum).colptr(begin), - predictors.n_rows, effectiveBatchSize, false, true))); + Forward(arma::mat(predictors.slice(seqNum).colptr(begin), + predictors.n_rows, effectiveBatchSize, false, true)); results.slice(seqNum).submat(0, begin, results.n_rows - 1, begin + effectiveBatchSize - 1) = boost::apply_visitor(outputParameterVisitor, @@ -224,16 +262,16 @@ double RNN::Evaluate( // Wrap a matrix around our data to avoid a copy. arma::mat stepData(predictors.slice(seqNum).colptr(begin), predictors.n_rows, batchSize, false, true); - Forward(std::move(stepData)); + Forward(stepData); if (!single) { responseSeq = seqNum; } - performance += outputLayer.Forward(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), - std::move(arma::mat(responses.slice(responseSeq).colptr(begin), - responses.n_rows, batchSize, false, true))); + performance += outputLayer.Forward(boost::apply_visitor( + outputParameterVisitor, network.back()), + arma::mat(responses.slice(responseSeq).colptr(begin), + responses.n_rows, batchSize, false, true)); } if (outputSize == 0) @@ -306,7 +344,7 @@ EvaluateWithGradient(const arma::mat& /* parameters */, // Wrap a matrix around our data to avoid a copy. arma::mat stepData(predictors.slice(seqNum).colptr(begin), predictors.n_rows, batchSize, false, true); - Forward(std::move(stepData)); + Forward(stepData); if (!single) { responseSeq = seqNum; @@ -314,14 +352,14 @@ EvaluateWithGradient(const arma::mat& /* parameters */, for (size_t l = 0; l < network.size(); ++l) { - boost::apply_visitor(SaveOutputParameterVisitor( - std::move(moduleOutputParameter)), network[l]); + boost::apply_visitor(SaveOutputParameterVisitor(moduleOutputParameter), + network[l]); } - performance += outputLayer.Forward(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), - std::move(arma::mat(responses.slice(responseSeq).colptr(begin), - responses.n_rows, batchSize, false, true))); + performance += outputLayer.Forward(boost::apply_visitor( + outputParameterVisitor, network.back()), + arma::mat(responses.slice(responseSeq).colptr(begin), + responses.n_rows, batchSize, false, true)); } if (outputSize == 0) @@ -344,8 +382,8 @@ EvaluateWithGradient(const arma::mat& /* parameters */, currentGradient.zeros(); for (size_t l = 0; l < network.size(); ++l) { - boost::apply_visitor(LoadOutputParameterVisitor( - std::move(moduleOutputParameter)), network[network.size() - 1 - l]); + boost::apply_visitor(LoadOutputParameterVisitor(moduleOutputParameter), + network[network.size() - 1 - l]); } if (single && seqNum > 0) @@ -354,24 +392,23 @@ EvaluateWithGradient(const arma::mat& /* parameters */, } else if (single && seqNum == 0) { - outputLayer.Backward(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), - std::move(arma::mat(responses.slice(0).colptr(begin), - responses.n_rows, batchSize, false, true)), std::move(error)); + outputLayer.Backward(boost::apply_visitor( + outputParameterVisitor, network.back()), + arma::mat(responses.slice(0).colptr(begin), + responses.n_rows, batchSize, false, true), error); } else { - outputLayer.Backward(std::move(boost::apply_visitor( - outputParameterVisitor, network.back())), - std::move(arma::mat( - responses.slice(effectiveRho - seqNum - 1).colptr(begin), - responses.n_rows, batchSize, false, true)), std::move(error)); + outputLayer.Backward(boost::apply_visitor( + outputParameterVisitor, network.back()), + arma::mat(responses.slice(effectiveRho - seqNum - 1).colptr(begin), + responses.n_rows, batchSize, false, true), error); } Backward(); - Gradient(std::move( + Gradient( arma::mat(predictors.slice(effectiveRho - seqNum - 1).colptr(begin), - predictors.n_rows, batchSize, false, true))); + predictors.n_rows, batchSize, false, true)); gradient += currentGradient; } @@ -444,25 +481,25 @@ void RNN& layer : network) { - offset += boost::apply_visitor(GradientSetVisitor(std::move(gradient), - offset), layer); + offset += boost::apply_visitor(GradientSetVisitor(gradient, offset), layer); } } template +template void RNN::Forward(arma::mat&& input) + CustomLayers...>::Forward(const InputType& input) { - boost::apply_visitor(ForwardVisitor(std::move(input), std::move( - boost::apply_visitor(outputParameterVisitor, network.front()))), + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network.front())), network.front()); for (size_t i = 1; i < network.size(); ++i) { boost::apply_visitor(ForwardVisitor( - std::move(boost::apply_visitor(outputParameterVisitor, network[i - 1])), - std::move(boost::apply_visitor(outputParameterVisitor, network[i]))), + boost::apply_visitor(outputParameterVisitor, network[i - 1]), + boost::apply_visitor(outputParameterVisitor, network[i])), network[i]); } } @@ -472,17 +509,17 @@ template::Backward() { boost::apply_visitor(BackwardVisitor( - std::move(boost::apply_visitor(outputParameterVisitor, network.back())), - std::move(error), std::move(boost::apply_visitor(deltaVisitor, - network.back()))), network.back()); + boost::apply_visitor(outputParameterVisitor, network.back()), + error, boost::apply_visitor(deltaVisitor, + network.back())), network.back()); for (size_t i = 2; i < network.size(); ++i) { boost::apply_visitor(BackwardVisitor( - std::move(boost::apply_visitor(outputParameterVisitor, - network[network.size() - i])), std::move(boost::apply_visitor( - deltaVisitor, network[network.size() - i + 1])), std::move( - boost::apply_visitor(deltaVisitor, network[network.size() - i]))), + boost::apply_visitor(outputParameterVisitor, + network[network.size() - i]), boost::apply_visitor( + deltaVisitor, network[network.size() - i + 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i])), network[network.size() - i]); } } @@ -491,16 +528,16 @@ template template void RNN::Gradient(InputType&& input) + CustomLayers...>::Gradient(const InputType& input) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move( - boost::apply_visitor(deltaVisitor, network[1]))), network.front()); + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, network[1])), network.front()); for (size_t i = 1; i < network.size() - 1; ++i) { boost::apply_visitor(GradientVisitor( - std::move(boost::apply_visitor(outputParameterVisitor, network[i - 1])), - std::move(boost::apply_visitor(deltaVisitor, network[i + 1]))), + boost::apply_visitor(outputParameterVisitor, network[i - 1]), + boost::apply_visitor(deltaVisitor, network[i + 1])), network[i]); } } @@ -544,8 +581,8 @@ void RNN::serialize( size_t offset = 0; for (LayerTypes& layer : network) { - offset += boost::apply_visitor(WeightSetVisitor(std::move(parameter), - offset), layer); + offset += boost::apply_visitor(WeightSetVisitor(parameter, offset), + layer); boost::apply_visitor(resetVisitor, layer); } diff --git a/src/mlpack/methods/ann/visitor/backward_visitor.hpp b/src/mlpack/methods/ann/visitor/backward_visitor.hpp index a893f35d16..b7755e42cb 100644 --- a/src/mlpack/methods/ann/visitor/backward_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/backward_visitor.hpp @@ -30,11 +30,15 @@ class BackwardVisitor : public boost::static_visitor public: //! Execute the Backward() function given the input, error and delta //! parameter. - BackwardVisitor(arma::mat&& input, arma::mat&& error, arma::mat&& delta); + BackwardVisitor(const arma::mat& input, + const arma::mat& error, + arma::mat& delta); //! Execute the Backward() function for the layer with the specified index. - BackwardVisitor(arma::mat&& input, arma::mat&& error, arma::mat&& delta, - const size_t index); + BackwardVisitor(const arma::mat& input, + const arma::mat& error, + arma::mat& delta, + const size_t index); //! Execute the Backward() function. template @@ -44,13 +48,13 @@ class BackwardVisitor : public boost::static_visitor private: //! The input parameter set. - arma::mat&& input; + const arma::mat& input; //! The error parameter. - arma::mat&& error; + const arma::mat& error; //! The delta parameter. - arma::mat&& delta; + arma::mat& delta; //! The index of the layer to run. size_t index; diff --git a/src/mlpack/methods/ann/visitor/backward_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/backward_visitor_impl.hpp index ef449e45bd..e49f3c2e41 100644 --- a/src/mlpack/methods/ann/visitor/backward_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/backward_visitor_impl.hpp @@ -19,25 +19,25 @@ namespace mlpack { namespace ann { //! BackwardVisitor visitor class. -inline BackwardVisitor::BackwardVisitor(arma::mat&& input, - arma::mat&& error, - arma::mat&& delta) : - input(std::move(input)), - error(std::move(error)), - delta(std::move(delta)), +inline BackwardVisitor::BackwardVisitor(const arma::mat& input, + const arma::mat& error, + arma::mat& delta) : + input(input), + error(error), + delta(delta), index(0), hasIndex(false) { /* Nothing to do here. */ } -inline BackwardVisitor::BackwardVisitor(arma::mat&& input, - arma::mat&& error, - arma::mat&& delta, +inline BackwardVisitor::BackwardVisitor(const arma::mat& input, + const arma::mat& error, + arma::mat& delta, const size_t index) : - input(std::move(input)), - error(std::move(error)), - delta(std::move(delta)), + input(input), + error(error), + delta(delta), index(index), hasIndex(true) { @@ -60,7 +60,7 @@ inline typename std::enable_if< !HasRunCheck::value, void>::type BackwardVisitor::LayerBackward(T* layer, arma::mat& /* input */) const { - layer->Backward(std::move(input), std::move(error), std::move(delta)); + layer->Backward(input, error, delta); } template @@ -70,13 +70,11 @@ BackwardVisitor::LayerBackward(T* layer, arma::mat& /* input */) const { if (!hasIndex) { - layer->Backward(std::move(input), std::move(error), - std::move(delta)); + layer->Backward(input, error, delta); } else { - layer->Backward(std::move(input), std::move(error), - std::move(delta), index); + layer->Backward(input, error, delta, index); } } diff --git a/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp b/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp index d0c4c75706..e67f4a1e02 100644 --- a/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp @@ -27,7 +27,7 @@ class BiasSetVisitor : public boost::static_visitor { public: //! Update the bias parameters given the parameters' set and offset. - BiasSetVisitor(arma::mat&& weight, const size_t offset = 0); + BiasSetVisitor(arma::mat& weight, const size_t offset = 0); //! Update the parameters' set. template @@ -37,7 +37,7 @@ class BiasSetVisitor : public boost::static_visitor private: //! The parameters' set. - arma::mat&& weight; + arma::mat& weight; //! The parameters' offset. const size_t offset; diff --git a/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp index 46e0913ad1..73051c80d8 100644 --- a/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp @@ -19,8 +19,8 @@ namespace mlpack { namespace ann { //! BiasSetVisitor visitor class. -inline BiasSetVisitor::BiasSetVisitor(arma::mat&& weight, const size_t offset) : - weight(std::move(weight)), +inline BiasSetVisitor::BiasSetVisitor(arma::mat& weight, const size_t offset) : + weight(weight), offset(offset) { /* Nothing to do here. */ @@ -57,7 +57,7 @@ BiasSetVisitor::LayerSize(T* layer) const for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(BiasSetVisitor( - std::move(weight), modelOffset + offset), layer->Model()[i]); + weight, modelOffset + offset), layer->Model()[i]); } return modelOffset; @@ -89,7 +89,7 @@ BiasSetVisitor::LayerSize(T* layer) const for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(BiasSetVisitor( - std::move(weight), modelOffset + offset), layer->Model()[i]); + weight, modelOffset + offset), layer->Model()[i]); } return modelOffset; diff --git a/src/mlpack/methods/ann/visitor/delete_visitor.hpp b/src/mlpack/methods/ann/visitor/delete_visitor.hpp index 6dbcad293d..5ffd96d775 100644 --- a/src/mlpack/methods/ann/visitor/delete_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/delete_visitor.hpp @@ -27,9 +27,17 @@ namespace ann { class DeleteVisitor : public boost::static_visitor { public: - //! Execute the destructor. + //! Execute the destructor if the layer does not hold layers internally. template - void operator()(LayerType* layer) const; + typename std::enable_if< + !HasModelCheck::value, void>::type + operator()(LayerType* layer) const; + + //! Execute the destructor if the layer does hold layers internally. + template + typename std::enable_if< + HasModelCheck::value, void>::type + operator()(LayerType* layer) const; void operator()(MoreTypes layer) const; }; diff --git a/src/mlpack/methods/ann/visitor/delete_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/delete_visitor_impl.hpp index dbf6e2a7c2..e2e82d5880 100644 --- a/src/mlpack/methods/ann/visitor/delete_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/delete_visitor_impl.hpp @@ -20,12 +20,28 @@ namespace ann { //! DeleteVisitor visitor class. template -inline void DeleteVisitor::operator()(LayerType* layer) const +inline typename std::enable_if< + !HasModelCheck::value, void>::type +DeleteVisitor::operator()(LayerType* layer) const { if (layer) delete layer; } +template +inline typename std::enable_if< + HasModelCheck::value, void>::type +DeleteVisitor::operator()(LayerType* layer) const +{ + if (layer) + { + for (size_t i = 0; i < layer->Model().size(); ++i) + boost::apply_visitor(DeleteVisitor(), layer->Model()[i]); + + delete layer; + } +} + inline void DeleteVisitor::operator()(MoreTypes layer) const { layer.apply_visitor(*this); diff --git a/src/mlpack/methods/ann/visitor/forward_visitor.hpp b/src/mlpack/methods/ann/visitor/forward_visitor.hpp index c28ab3ad34..e2c181a5d6 100644 --- a/src/mlpack/methods/ann/visitor/forward_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/forward_visitor.hpp @@ -29,7 +29,7 @@ class ForwardVisitor : public boost::static_visitor { public: //! Execute the Forward() function given the input and output parameter. - ForwardVisitor(arma::mat&& input, arma::mat&& output); + ForwardVisitor(const arma::mat& input, arma::mat& output); //! Execute the Forward() function. template @@ -39,10 +39,10 @@ class ForwardVisitor : public boost::static_visitor private: //! The input parameter set. - arma::mat&& input; + const arma::mat& input; //! The output parameter set. - arma::mat&& output; + arma::mat& output; }; } // namespace ann diff --git a/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp index ffddac9bb7..44baacdc75 100644 --- a/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp @@ -19,9 +19,9 @@ namespace mlpack { namespace ann { //! ForwardVisitor visitor class. -inline ForwardVisitor::ForwardVisitor(arma::mat&& input, arma::mat&& output) : - input(std::move(input)), - output(std::move(output)) +inline ForwardVisitor::ForwardVisitor(const arma::mat& input, arma::mat& output) : + input(input), + output(output) { /* Nothing to do here. */ } @@ -29,7 +29,7 @@ inline ForwardVisitor::ForwardVisitor(arma::mat&& input, arma::mat&& output) : template inline void ForwardVisitor::operator()(LayerType* layer) const { - layer->Forward(std::move(input), std::move(output)); + layer->Forward(input, output); } inline void ForwardVisitor::operator()(MoreTypes layer) const diff --git a/src/mlpack/methods/ann/visitor/gradient_set_visitor.hpp b/src/mlpack/methods/ann/visitor/gradient_set_visitor.hpp index 13992cd89d..62a19c8ae0 100644 --- a/src/mlpack/methods/ann/visitor/gradient_set_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/gradient_set_visitor.hpp @@ -27,7 +27,7 @@ class GradientSetVisitor : public boost::static_visitor { public: //! Update the gradient parameter given the gradient set. - GradientSetVisitor(arma::mat&& gradient, size_t offset = 0); + GradientSetVisitor(arma::mat& gradient, size_t offset = 0); //! Update the gradient parameter. template @@ -37,7 +37,7 @@ class GradientSetVisitor : public boost::static_visitor private: //! The gradient set. - arma::mat&& gradient; + arma::mat& gradient; //! The gradient offset. size_t offset; diff --git a/src/mlpack/methods/ann/visitor/gradient_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/gradient_set_visitor_impl.hpp index 863c68df14..677578182c 100644 --- a/src/mlpack/methods/ann/visitor/gradient_set_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/gradient_set_visitor_impl.hpp @@ -19,9 +19,9 @@ namespace mlpack { namespace ann { //! GradientSetVisitor visitor class. -inline GradientSetVisitor::GradientSetVisitor(arma::mat&& gradient, +inline GradientSetVisitor::GradientSetVisitor(arma::mat& gradient, size_t offset) : - gradient(std::move(gradient)), + gradient(gradient), offset(offset) { /* Nothing to do here. */ @@ -60,7 +60,7 @@ GradientSetVisitor::LayerGradients(T* layer, arma::mat& /* input */) const for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(GradientSetVisitor( - std::move(gradient), modelOffset + offset), layer->Model()[i]); + gradient, modelOffset + offset), layer->Model()[i]); } return modelOffset; @@ -79,7 +79,7 @@ GradientSetVisitor::LayerGradients(T* layer, arma::mat& /* input */) const for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(GradientSetVisitor( - std::move(gradient), modelOffset + offset), layer->Model()[i]); + gradient, modelOffset + offset), layer->Model()[i]); } return modelOffset; diff --git a/src/mlpack/methods/ann/visitor/gradient_update_visitor.hpp b/src/mlpack/methods/ann/visitor/gradient_update_visitor.hpp index 2913aaf977..6eec2a777b 100644 --- a/src/mlpack/methods/ann/visitor/gradient_update_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/gradient_update_visitor.hpp @@ -27,7 +27,7 @@ class GradientUpdateVisitor : public boost::static_visitor { public: //! Update the gradient parameter given the gradient set. - GradientUpdateVisitor(arma::mat&& gradient, size_t offset = 0); + GradientUpdateVisitor(arma::mat& gradient, size_t offset = 0); //! Update the gradient parameter. template @@ -37,7 +37,7 @@ class GradientUpdateVisitor : public boost::static_visitor private: //! The gradient set. - arma::mat&& gradient; + arma::mat& gradient; //! The gradient offset. size_t offset; diff --git a/src/mlpack/methods/ann/visitor/gradient_update_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/gradient_update_visitor_impl.hpp index 6c77c9ae27..1c31b5a6f8 100644 --- a/src/mlpack/methods/ann/visitor/gradient_update_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/gradient_update_visitor_impl.hpp @@ -19,9 +19,9 @@ namespace mlpack { namespace ann { //! GradientUpdateVisitor visitor class. -inline GradientUpdateVisitor::GradientUpdateVisitor(arma::mat&& gradient, +inline GradientUpdateVisitor::GradientUpdateVisitor(arma::mat& gradient, size_t offset) : - gradient(std::move(gradient)), + gradient(gradient), offset(offset) { /* Nothing to do here. */ @@ -63,7 +63,7 @@ GradientUpdateVisitor::LayerGradients(T* layer, arma::mat& /* input */) const for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(GradientUpdateVisitor( - std::move(gradient), modelOffset + offset), layer->Model()[i]); + gradient, modelOffset + offset), layer->Model()[i]); } return modelOffset; @@ -85,7 +85,7 @@ GradientUpdateVisitor::LayerGradients(T* layer, arma::mat& /* input */) const for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(GradientUpdateVisitor( - std::move(gradient), modelOffset + offset), layer->Model()[i]); + gradient, modelOffset + offset), layer->Model()[i]); } return modelOffset; diff --git a/src/mlpack/methods/ann/visitor/gradient_visitor.hpp b/src/mlpack/methods/ann/visitor/gradient_visitor.hpp index e95099cb68..64ab4d7fd1 100644 --- a/src/mlpack/methods/ann/visitor/gradient_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/gradient_visitor.hpp @@ -30,10 +30,12 @@ class GradientVisitor : public boost::static_visitor public: //! Executes the Gradient() method of the given module using the input and //! delta parameter. - GradientVisitor(arma::mat&& input, arma::mat&& delta); + GradientVisitor(const arma::mat& input, const arma::mat& delta); //! Executes the Gradient() method for the layer with the specified index. - GradientVisitor(arma::mat&& input, arma::mat&& delta, const size_t index); + GradientVisitor(const arma::mat& input, + const arma::mat& delta, + const size_t index); //! Executes the Gradient() method. template @@ -43,10 +45,10 @@ class GradientVisitor : public boost::static_visitor private: //! The input set. - arma::mat&& input; + const arma::mat& input; //! The delta parameter. - arma::mat&& delta; + const arma::mat& delta; //! Index of the layer to run. size_t index; diff --git a/src/mlpack/methods/ann/visitor/gradient_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/gradient_visitor_impl.hpp index 77852e80f1..02b40d6db9 100644 --- a/src/mlpack/methods/ann/visitor/gradient_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/gradient_visitor_impl.hpp @@ -19,19 +19,21 @@ namespace mlpack { namespace ann { //! GradientVisitor visitor class. -inline GradientVisitor::GradientVisitor(arma::mat&& input, arma::mat&& delta) : - input(std::move(input)), - delta(std::move(delta)), +inline GradientVisitor::GradientVisitor(const arma::mat& input, + const arma::mat& delta) : + input(input), + delta(delta), index(0), hasIndex(false) { /* Nothing to do here. */ } -inline GradientVisitor::GradientVisitor(arma::mat&& input, arma::mat&& delta, +inline GradientVisitor::GradientVisitor(const arma::mat& input, + const arma::mat& delta, const size_t index) : - input(std::move(input)), - delta(std::move(delta)), + input(input), + delta(delta), index(index), hasIndex(true) { @@ -55,8 +57,7 @@ inline typename std::enable_if< !HasRunCheck::value, void>::type GradientVisitor::LayerGradients(T* layer, arma::mat& /* input */) const { - layer->Gradient(std::move(input), std::move(delta), - std::move(layer->Gradient())); + layer->Gradient(input, delta, layer->Gradient()); } template @@ -67,13 +68,11 @@ GradientVisitor::LayerGradients(T* layer, arma::mat& /* input */) const { if (!hasIndex) { - layer->Gradient(std::move(input), std::move(delta), - std::move(layer->Gradient())); + layer->Gradient(input, delta, layer->Gradient()); } else { - layer->Gradient(std::move(input), std::move(delta), - std::move(layer->Gradient()), index); + layer->Gradient(input, delta, layer->Gradient(), index); } } diff --git a/src/mlpack/methods/ann/visitor/load_output_parameter_visitor.hpp b/src/mlpack/methods/ann/visitor/load_output_parameter_visitor.hpp index e8cd371fe2..2ba506f78c 100644 --- a/src/mlpack/methods/ann/visitor/load_output_parameter_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/load_output_parameter_visitor.hpp @@ -29,7 +29,7 @@ class LoadOutputParameterVisitor : public boost::static_visitor { public: //! Restore the output parameter given a parameter set. - LoadOutputParameterVisitor(std::vector&& parameter); + LoadOutputParameterVisitor(std::vector& parameter); //! Restore the output parameter. template @@ -39,7 +39,7 @@ class LoadOutputParameterVisitor : public boost::static_visitor private: //! The parameter set. - std::vector&& parameter; + std::vector& parameter; //! Restore the output parameter for a module which doesn't implement the //! Model() function. diff --git a/src/mlpack/methods/ann/visitor/load_output_parameter_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/load_output_parameter_visitor_impl.hpp index 427fda1d45..a395ad12dc 100644 --- a/src/mlpack/methods/ann/visitor/load_output_parameter_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/load_output_parameter_visitor_impl.hpp @@ -20,7 +20,7 @@ namespace ann { //! LoadOutputParameterVisitor visitor class. inline LoadOutputParameterVisitor::LoadOutputParameterVisitor( - std::vector&& parameter) : parameter(std::move(parameter)) + std::vector& parameter) : parameter(parameter) { /* Nothing to do here. */ } @@ -52,7 +52,7 @@ LoadOutputParameterVisitor::OutputParameter(T* layer) const { for (size_t i = 0; i < layer->Model().size(); ++i) { - boost::apply_visitor(LoadOutputParameterVisitor(std::move(parameter)), + boost::apply_visitor(LoadOutputParameterVisitor(parameter), layer->Model()[layer->Model().size() - i - 1]); } diff --git a/src/mlpack/methods/ann/visitor/parameters_set_visitor.hpp b/src/mlpack/methods/ann/visitor/parameters_set_visitor.hpp index 87f0fac275..bedad4b9aa 100644 --- a/src/mlpack/methods/ann/visitor/parameters_set_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/parameters_set_visitor.hpp @@ -28,7 +28,7 @@ class ParametersSetVisitor : public boost::static_visitor { public: //! Update the parameters set given the parameters matrix. - ParametersSetVisitor(arma::mat&& parameters); + ParametersSetVisitor(arma::mat& parameters); //! Update the parameters set. template @@ -38,7 +38,7 @@ class ParametersSetVisitor : public boost::static_visitor private: //! The parameters set. - arma::mat&& parameters; + arma::mat& parameters; //! Do not update the parameters set if the module doesn't implement the //! Parameters() function. diff --git a/src/mlpack/methods/ann/visitor/parameters_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/parameters_set_visitor_impl.hpp index d7fb7a09ea..a1bbe87dbd 100644 --- a/src/mlpack/methods/ann/visitor/parameters_set_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/parameters_set_visitor_impl.hpp @@ -19,8 +19,8 @@ namespace mlpack { namespace ann { //! ParametersSetVisitor visitor class. -inline ParametersSetVisitor::ParametersSetVisitor(arma::mat&& parameters) : - parameters(std::move(parameters)) +inline ParametersSetVisitor::ParametersSetVisitor(arma::mat& parameters) : + parameters(parameters) { /* Nothing to do here. */ } diff --git a/src/mlpack/methods/ann/visitor/parameters_visitor.hpp b/src/mlpack/methods/ann/visitor/parameters_visitor.hpp index 58c207ebe5..2a63586dc8 100644 --- a/src/mlpack/methods/ann/visitor/parameters_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/parameters_visitor.hpp @@ -29,7 +29,7 @@ class ParametersVisitor : public boost::static_visitor { public: //! Store the parameters set into the given parameters matrix. - ParametersVisitor(arma::mat&& parameters); + ParametersVisitor(arma::mat& parameters); //! Set the parameters set. template @@ -39,7 +39,7 @@ class ParametersVisitor : public boost::static_visitor private: //! The parameters set. - arma::mat&& parameters; + arma::mat& parameters; //! Do not set the parameters set if the module doesn't implement the //! Parameters() function. diff --git a/src/mlpack/methods/ann/visitor/parameters_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/parameters_visitor_impl.hpp index 0b9a680337..957dcfc7a9 100644 --- a/src/mlpack/methods/ann/visitor/parameters_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/parameters_visitor_impl.hpp @@ -19,8 +19,8 @@ namespace mlpack { namespace ann { //! ParametersVisitor visitor class. -inline ParametersVisitor::ParametersVisitor(arma::mat&& parameters) : - parameters(std::move(parameters)) +inline ParametersVisitor::ParametersVisitor(arma::mat& parameters) : + parameters(parameters) { /* Nothing to do here. */ } diff --git a/src/mlpack/methods/ann/visitor/save_output_parameter_visitor.hpp b/src/mlpack/methods/ann/visitor/save_output_parameter_visitor.hpp index ed285e560e..dbfd908b08 100644 --- a/src/mlpack/methods/ann/visitor/save_output_parameter_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/save_output_parameter_visitor.hpp @@ -28,7 +28,7 @@ class SaveOutputParameterVisitor : public boost::static_visitor { public: //! Save the output parameter into the given parameter set. - SaveOutputParameterVisitor(std::vector&& parameter); + SaveOutputParameterVisitor(std::vector& parameter); //! Save the output parameter. template @@ -38,7 +38,7 @@ class SaveOutputParameterVisitor : public boost::static_visitor private: //! The parameter set. - std::vector&& parameter; + std::vector& parameter; //! Save the output parameter for a module which doesn't implement the //! Model() function. diff --git a/src/mlpack/methods/ann/visitor/save_output_parameter_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/save_output_parameter_visitor_impl.hpp index 725dcc49b8..a73719abd6 100644 --- a/src/mlpack/methods/ann/visitor/save_output_parameter_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/save_output_parameter_visitor_impl.hpp @@ -20,7 +20,7 @@ namespace ann { //! SaveOutputParameterVisitor visitor class. inline SaveOutputParameterVisitor::SaveOutputParameterVisitor( - std::vector&& parameter) : parameter(std::move(parameter)) + std::vector& parameter) : parameter(parameter) { /* Nothing to do here. */ } @@ -53,7 +53,7 @@ SaveOutputParameterVisitor::OutputParameter(T* layer) const for (size_t i = 0; i < layer->Model().size(); ++i) { - boost::apply_visitor(SaveOutputParameterVisitor(std::move(parameter)), + boost::apply_visitor(SaveOutputParameterVisitor(parameter), layer->Model()[i]); } } diff --git a/src/mlpack/methods/ann/visitor/weight_set_visitor.hpp b/src/mlpack/methods/ann/visitor/weight_set_visitor.hpp index 62d93c3290..4bbad8cc4c 100644 --- a/src/mlpack/methods/ann/visitor/weight_set_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/weight_set_visitor.hpp @@ -27,7 +27,7 @@ class WeightSetVisitor : public boost::static_visitor { public: //! Update the parameters given the parameters set and offset. - WeightSetVisitor(arma::mat&& weight, const size_t offset = 0); + WeightSetVisitor(arma::mat& weight, const size_t offset = 0); //! Update the parameters set. template @@ -37,7 +37,7 @@ class WeightSetVisitor : public boost::static_visitor private: //! The parameters set. - arma::mat&& weight; + arma::mat& weight; //! The parameters offset. const size_t offset; diff --git a/src/mlpack/methods/ann/visitor/weight_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/weight_set_visitor_impl.hpp index 9be9593647..14edbc54b3 100644 --- a/src/mlpack/methods/ann/visitor/weight_set_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/weight_set_visitor_impl.hpp @@ -19,9 +19,9 @@ namespace mlpack { namespace ann { //! WeightSetVisitor visitor class. -inline WeightSetVisitor::WeightSetVisitor(arma::mat&& weight, +inline WeightSetVisitor::WeightSetVisitor(arma::mat& weight, const size_t offset) : - weight(std::move(weight)), + weight(weight), offset(offset) { /* Nothing to do here. */ @@ -30,7 +30,7 @@ inline WeightSetVisitor::WeightSetVisitor(arma::mat&& weight, template inline size_t WeightSetVisitor::operator()(LayerType* layer) const { - return LayerSize(layer, std::move(layer->OutputParameter())); + return LayerSize(layer, layer->OutputParameter()); } inline size_t WeightSetVisitor::operator()(MoreTypes layer) const @@ -57,7 +57,7 @@ WeightSetVisitor::LayerSize(T* layer, P&& /*output */) const for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(WeightSetVisitor( - std::move(weight), modelOffset + offset), layer->Model()[i]); + weight, modelOffset + offset), layer->Model()[i]); } return modelOffset; @@ -88,7 +88,7 @@ WeightSetVisitor::LayerSize(T* layer, P&& /* output */) const for (size_t i = 0; i < layer->Model().size(); ++i) { modelOffset += boost::apply_visitor(WeightSetVisitor( - std::move(weight), modelOffset + offset), layer->Model()[i]); + weight, modelOffset + offset), layer->Model()[i]); } return modelOffset; diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp index 49faf450fc..d609bd51e9 100644 --- a/src/mlpack/methods/cf/cf_main.cpp +++ b/src/mlpack/methods/cf/cf_main.cpp @@ -223,14 +223,14 @@ void ComputeRecommendations(CFModel* cf, const size_t numRecs, arma::Mat& recommendations) { - // Verifying the Interpolation algorithms + // Verify the Interpolation algorithms. RequireParamInSet("interpolation", { "average", "regression", "similarity" }, true, "unknown interpolation algorithm"); - // Taking Interpolation Alternatives + // Taking Interpolation Alternatives const string interpolationAlgorithm = CLI::GetParam("interpolation"); - // Determining the Interpolation Algorithm + // Determining the Interpolation Algorithm if (interpolationAlgorithm == "average") { ComputeRecommendations @@ -382,6 +382,11 @@ void PerformAction(arma::mat& dataset, const double minResidue) { const size_t neighborhood = (size_t) CLI::GetParam("neighborhood"); + + // Make sure the normalization strategy is valid. + RequireParamInSet("normalization", { "overall_mean", "item_mean", + "user_mean", "z_score", "none" }, true, "unknown normalization type"); + CFModel* c = new CFModel(); const string normalizationType = CLI::GetParam("normalization"); @@ -390,7 +395,16 @@ void PerformAction(arma::mat& dataset, maxIterations, minResidue, CLI::HasParam("iteration_only_termination"), normalizationType); - PerformAction(c); + try + { + PerformAction(c); + } + catch (std::exception& e) + { + // Clean the memory before throwing completely. + delete c; + throw; + } } void AssembleFactorizerType(const std::string& algorithm, diff --git a/src/mlpack/methods/fastmks/fastmks_main.cpp b/src/mlpack/methods/fastmks/fastmks_main.cpp index 31a575c1b2..b9a96118aa 100644 --- a/src/mlpack/methods/fastmks/fastmks_main.cpp +++ b/src/mlpack/methods/fastmks/fastmks_main.cpp @@ -128,6 +128,12 @@ static void mlpackMain() "number of maximum kernels must be greater than 0"); } + if (CLI::HasParam("base")) + { + RequireParamValue("base", [](double x) { return x > 1.0; }, true, + "base must be greater than or equal to 1!"); + } + // Naive mode overrides single mode. ReportIgnoredParam({{ "naive", true }}, "single"); @@ -223,12 +229,32 @@ static void mlpackMain() Log::Info << "Loaded query data (" << queryData.n_rows << " x " << queryData.n_cols << ")." << endl; - model->Search(queryData, (size_t) CLI::GetParam("k"), indices, - kernels, base); + try + { + model->Search(queryData, (size_t) CLI::GetParam("k"), indices, + kernels, base); + } + catch (std::invalid_argument& e) + { + // Delete the memory, if needed. + if (CLI::HasParam("reference")) + delete model; + throw; + } } else { - model->Search((size_t) CLI::GetParam("k"), indices, kernels); + try + { + model->Search((size_t) CLI::GetParam("k"), indices, kernels); + } + catch (std::invalid_argument& e) + { + // Delete the memory, if needed. + if (CLI::HasParam("reference")) + delete model; + throw e; + } } // Save output. diff --git a/src/mlpack/methods/gmm/gmm_train_main.cpp b/src/mlpack/methods/gmm/gmm_train_main.cpp index bfe29ae562..712c1e957b 100644 --- a/src/mlpack/methods/gmm/gmm_train_main.cpp +++ b/src/mlpack/methods/gmm/gmm_train_main.cpp @@ -188,10 +188,6 @@ static void mlpackMain() << " model (given with " << PRINT_PARAM_STRING("input_model") << " has dimensionality " << gmm->Dimensionality() << "!" << endl; } - else - { - gmm = new GMM(size_t(gaussians), dataPoints.n_rows); - } // Gather parameters for EMFit object. const size_t maxIterations = (size_t) CLI::GetParam("max_iterations"); @@ -212,6 +208,11 @@ static void mlpackMain() return x > 0.0 && x <= 1.0; }, true, "percentage to sample must be " "be greater than 0.0 and less than or equal to 1.0"); + // Initialize the GMM if needed. (We didn't do this earlier, because + // RequireParamValue() would leak the memory if the check failed.) + if (!CLI::HasParam("input_model")) + gmm = new GMM(size_t(gaussians), dataPoints.n_rows); + const int samplings = CLI::GetParam("samplings"); const double percentage = CLI::GetParam("percentage"); @@ -274,6 +275,10 @@ static void mlpackMain() } else { + // Initialize the GMM if needed. + if (!CLI::HasParam("input_model")) + gmm = new GMM(size_t(gaussians), dataPoints.n_rows); + // Depending on the value of forcePositive and diagonalCovariance, we have // to use different types. if (diagonalCovariance) diff --git a/src/mlpack/methods/hmm/hmm_train_main.cpp b/src/mlpack/methods/hmm/hmm_train_main.cpp index 5769a0f21e..e748d7d7e9 100644 --- a/src/mlpack/methods/hmm/hmm_train_main.cpp +++ b/src/mlpack/methods/hmm/hmm_train_main.cpp @@ -515,16 +515,26 @@ static void mlpackMain() if (CLI::HasParam("input_model")) { hmm = CLI::GetParam("input_model"); + + hmm->PerformAction>(&trainSeq); } else { // We need to initialize the model. hmm = new HMMModel(typeId); - hmm->PerformAction>(&trainSeq); - } - // Train the model. - hmm->PerformAction>(&trainSeq); + // Catch any exceptions so that we can clean the model if needed. + try + { + hmm->PerformAction>(&trainSeq); + hmm->PerformAction>(&trainSeq); + } + catch (std::exception& e) + { + delete hmm; + throw; + } + } // If necessary, save the output. CLI::GetParam("output_model") = hmm; diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp index d5e5d33c57..955916f068 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp @@ -121,6 +121,8 @@ class HoeffdingTree * @param dimensionMappings Mappings from dimension indices to positions in * numeric and categorical split vectors. If left NULL, a new one will * be created. + * @param copyDatasetInfo If true, then a copy of the datasetInfo will be + * made. */ HoeffdingTree(const data::DatasetInfo& datasetInfo, const size_t numClasses, @@ -133,7 +135,8 @@ class HoeffdingTree const NumericSplitType& numericSplitIn = NumericSplitType(0), std::unordered_map>* - dimensionMappings = NULL); + dimensionMappings = NULL, + const bool copyDatasetInfo = true); /** * Construct a Hoeffding tree with no data and no information. Be sure to diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp index c0f647fe4c..e1769e55b1 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp @@ -96,7 +96,8 @@ HoeffdingTree< categoricalSplitIn, const NumericSplitType& numericSplitIn, std::unordered_map>* - dimensionMappingsIn) : + dimensionMappingsIn, + const bool copyDatasetInfo) : dimensionMappings((dimensionMappingsIn != NULL) ? dimensionMappingsIn : new std::unordered_map>()), ownsMappings(dimensionMappingsIn == NULL), @@ -105,8 +106,9 @@ HoeffdingTree< maxSamples((maxSamples == 0) ? size_t(-1) : maxSamples), checkInterval(checkInterval), minSamples(minSamples), - datasetInfo(new data::DatasetInfo(datasetInfo)), - ownsInfo(true), + datasetInfo(copyDatasetInfo ? new data::DatasetInfo(datasetInfo) : + &datasetInfo), + ownsInfo(copyDatasetInfo), successProbability(successProbability), splitDimension(size_t(-1)), majorityClass(0), @@ -208,7 +210,18 @@ HoeffdingTree:: { // Copy each of the children. for (size_t i = 0; i < other.children.size(); ++i) + { children.push_back(new HoeffdingTree(*other.children[i])); + + // Delete copied datasetInfo and dimension mappings. + delete children[i]->datasetInfo; + children[i]->datasetInfo = this->datasetInfo; + children[i]->ownsInfo = false; + + delete children[i]->dimensionMappings; + children[i]->dimensionMappings = this->dimensionMappings; + children[i]->ownsMappings = false; + } } template(0, numClasses), - numericSplits[0], dimensionMappings)); + numericSplits[0], dimensionMappings, false)); } else if (numericSplits.size() == 0) { @@ -752,14 +765,14 @@ void HoeffdingTree< children.push_back(new HoeffdingTree(*datasetInfo, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplits[0], NumericSplitType(numClasses), - dimensionMappings)); + dimensionMappings, false)); } else { // Pass both splits that we already have. children.push_back(new HoeffdingTree(*datasetInfo, numClasses, successProbability, maxSamples, checkInterval, minSamples, - categoricalSplits[0], numericSplits[0], dimensionMappings)); + categoricalSplits[0], numericSplits[0], dimensionMappings, false)); } children[i]->MajorityClass() = childMajorities[i]; @@ -874,7 +887,8 @@ void HoeffdingTree< { // The child doesn't actually own its own DatasetInfo. We do. The same // applies for the dimension mappings. - children[i]->ownsInfo = false; + if (children[i]->datasetInfo == datasetInfo) + children[i]->ownsInfo = false; children[i]->ownsMappings = false; } diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model.cpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model.cpp index 5a27cffa3b..55635f50f1 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model.cpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model.cpp @@ -135,6 +135,12 @@ void HoeffdingTreeModel::BuildModel( const size_t bins, const size_t observationsBeforeBinning) { + // Clean memory, if needed. + delete giniHoeffdingTree; + delete giniBinaryTree; + delete infoHoeffdingTree; + delete infoBinaryTree; + // Depending on the type, create the tree. switch (type) { diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model.hpp index 7ab8754f3b..83ef9edf76 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model.hpp @@ -187,33 +187,13 @@ class HoeffdingTreeModel // Fake dataset info may be needed to create fake trees. data::DatasetInfo info; if (type == GINI_HOEFFDING) - { - // Create fake tree to load into if needed. - if (Archive::is_loading::value) - giniHoeffdingTree = new GiniHoeffdingTreeType(info, 1, 1); ar & BOOST_SERIALIZATION_NVP(giniHoeffdingTree); - } else if (type == GINI_BINARY) - { - // Create fake tree to load into if needed. - if (Archive::is_loading::value) - giniBinaryTree = new GiniBinaryTreeType(info, 1, 1); ar & BOOST_SERIALIZATION_NVP(giniBinaryTree); - } else if (type == INFO_HOEFFDING) - { - // Create fake tree to load into if needed. - if (Archive::is_loading::value) - infoHoeffdingTree = new InfoHoeffdingTreeType(info, 1, 1); ar & BOOST_SERIALIZATION_NVP(infoHoeffdingTree); - } else if (type == INFO_BINARY) - { - // Create fake tree to load into if needed. - if (Archive::is_loading::value) - infoBinaryTree = new InfoBinaryTreeType(info, 1, 1); ar & BOOST_SERIALIZATION_NVP(infoBinaryTree); - } } private: diff --git a/src/mlpack/methods/kde/kde_impl.hpp b/src/mlpack/methods/kde/kde_impl.hpp index 07a0ccfa0d..78f7c4fb4f 100644 --- a/src/mlpack/methods/kde/kde_impl.hpp +++ b/src/mlpack/methods/kde/kde_impl.hpp @@ -336,7 +336,16 @@ Evaluate(MatType querySet, arma::vec& estimations) std::vector oldFromNewQueries; Tree* queryTree = BuildTree(std::move(querySet), oldFromNewQueries); Timer::Stop("building_query_tree"); - this->Evaluate(queryTree, oldFromNewQueries, estimations); + try + { + this->Evaluate(queryTree, oldFromNewQueries, estimations); + } + catch (std::exception& e) + { + // Make sure we delete the query tree. + delete queryTree; + throw; + } delete queryTree; } else if (mode == SINGLE_TREE_MODE) diff --git a/src/mlpack/methods/kde/kde_main.cpp b/src/mlpack/methods/kde/kde_main.cpp index 32f4afedc5..87909d180b 100644 --- a/src/mlpack/methods/kde/kde_main.cpp +++ b/src/mlpack/methods/kde/kde_main.cpp @@ -314,6 +314,5 @@ static void mlpackMain() CLI::GetParam("predictions") = std::move(estimations); // Save model. - if (CLI::HasParam("output_model")) - CLI::GetParam("output_model") = kde; + CLI::GetParam("output_model") = kde; } diff --git a/src/mlpack/methods/lars/lars.cpp b/src/mlpack/methods/lars/lars.cpp index 527645a554..848f9ebb67 100644 --- a/src/mlpack/methods/lars/lars.cpp +++ b/src/mlpack/methods/lars/lars.cpp @@ -361,7 +361,7 @@ double LARS::Train(const arma::mat& matX, beta = betaPath.back(); Timer::Stop("lars_regression"); - return maxCorr; + return ComputeError(matX, y, !transposeData); } double LARS::Train(const arma::mat& data, @@ -538,3 +538,18 @@ void LARS::CholeskyDelete(const size_t colToKill) matUtriCholFactor.shed_row(n); } } + +double LARS::ComputeError(const arma::mat& matX, + const arma::rowvec& y, + const bool rowMajor) +{ + if (rowMajor) + { + return arma::accu(arma::pow(y - trans(matX * betaPath.back()), 2.0)); + } + + else + { + return arma::accu(arma::pow(y - betaPath.back().t() * matX, 2.0)); + } +} diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 2d9e336609..9d7aa24ada 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -183,7 +183,7 @@ class LARS * @param responses A vector of targets. * @param beta Vector to store the solution (the coefficients) in. * @param transposeData Set to false if the data is row-major. - * @return The final absolute maximum correlation. + * @return minimum cost error(||y-beta*X||2 is used to calculate error). */ double Train(const arma::mat& data, const arma::rowvec& responses, @@ -202,7 +202,7 @@ class LARS * @param responses A vector of targets. * @param transposeData Should be true if the input data is column-major and * false otherwise. - * @return The final absolute maximum correlation. + * @return minimum cost error(||y-beta*X||2 is used to calculate error). */ double Train(const arma::mat& data, const arma::rowvec& responses, @@ -244,6 +244,22 @@ class LARS template void serialize(Archive& ar, const unsigned int /* version */); + /** + * Compute cost error of the given data matrix using the + * currently-trained LARS model. Only ||y-beta*X||2 is used to calculate + * cost error. + * + * @param data Column-major input data (or row-major input data if rowMajor = + * true). + * @param responses A vector of targets. + * @param rowMajor Should be true if the data points matrix is row-major and + * false otherwise. + * @return The minimum cost error. + */ + double ComputeError(const arma::mat& matX, + const arma::rowvec& y, + const bool rowMajor = false); + private: //! Gram matrix. arma::mat matGramInternal; diff --git a/src/mlpack/methods/linear_svm/linear_svm_main.cpp b/src/mlpack/methods/linear_svm/linear_svm_main.cpp index 13385cb74c..ed210a5a62 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_main.cpp +++ b/src/mlpack/methods/linear_svm/linear_svm_main.cpp @@ -312,6 +312,13 @@ static void mlpackMain() model->svm.NumClasses() = numClasses; model->svm.FitIntercept() = intercept; + if (numClasses <= 1) + { + if (!CLI::HasParam("input_model")) + delete model; + throw std::invalid_argument("Given input data has only 1 class!"); + } + if (optimizerType == "lbfgs") { ens::L_BFGS lbfgsOpt; @@ -369,6 +376,9 @@ static void mlpackMain() // Checking the dimensionality of the test data. if (testSet.n_rows != trainingDimensionality) { + // Clean memory if needed. + if (!CLI::HasParam("input_model")) + delete model; Log::Fatal << "Test data dimensionality (" << testSet.n_rows << ") must " << "be the same as the dimensionality of the training data (" << trainingDimensionality << ")!" << endl; @@ -398,6 +408,8 @@ static void mlpackMain() if (testSet.n_cols != testLabels.n_elem) { + if (!CLI::HasParam("input_model")) + delete model; Log::Fatal << "Test data given with " << PRINT_PARAM_STRING("test") << " has " << testSet.n_cols << " points, but labels in " << PRINT_PARAM_STRING("test_labels") << " have " diff --git a/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp b/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp index d7a057a97f..1aaf645768 100644 --- a/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp +++ b/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp @@ -134,8 +134,6 @@ static void mlpackMain() LocalCoordinateCoding* lcc; if (CLI::HasParam("input_model")) lcc = CLI::GetParam("input_model"); - else - lcc = new LocalCoordinateCoding(0, 0.0); if (CLI::HasParam("training")) { @@ -160,6 +158,8 @@ static void mlpackMain() RequireParamValue("tolerance", [](double x) { return x > 0; }, 1, "Tolerance should be a positive real number"); + lcc = new LocalCoordinateCoding(0, 0.0); + lcc->Lambda() = CLI::GetParam("lambda"); lcc->Atoms() = (size_t) CLI::GetParam("atoms"); lcc->MaxIterations() = (size_t) CLI::GetParam("max_iterations"); @@ -181,14 +181,21 @@ static void mlpackMain() // Validate the size of the initial dictionary. if (lcc->Dictionary().n_cols != lcc->Atoms()) { - Log::Fatal << "The initial dictionary has " << lcc->Dictionary().n_cols + const size_t dictionarySize = lcc->Dictionary().n_cols; + const size_t atoms = lcc->Atoms(); + if (!CLI::HasParam("input_model")) + delete lcc; + Log::Fatal << "The initial dictionary has " << dictionarySize << " atoms, but the number of atoms was specified to be " - << lcc->Atoms() << "!" << endl; + << atoms << "!" << endl; } if (lcc->Dictionary().n_rows != matX.n_rows) { - Log::Fatal << "The initial dictionary has " << lcc->Dictionary().n_rows + const size_t dictionaryDimension = lcc->Dictionary().n_rows; + if (!CLI::HasParam("input_model")) + delete lcc; + Log::Fatal << "The initial dictionary has " << dictionaryDimension << " dimensions, but the data has " << matX.n_rows << " dimensions!" << endl; } @@ -209,10 +216,15 @@ static void mlpackMain() mat matY = std::move(CLI::GetParam("test")); if (matY.n_rows != lcc->Dictionary().n_rows) + { + const size_t dictionaryDimension = lcc->Dictionary().n_rows; + if (!CLI::HasParam("input_model")) + delete lcc; Log::Fatal << "Model was trained with a dimensionality of " - << lcc->Dictionary().n_rows << ", but data in test file " + << dictionaryDimension << ", but data in test file " << CLI::GetPrintableParam("test") << " has a dimensionality of " << matY.n_rows << "!" << endl; + } // Normalize each point if the user asked for it. if (CLI::HasParam("normalize")) diff --git a/src/mlpack/methods/lsh/lsh_main.cpp b/src/mlpack/methods/lsh/lsh_main.cpp index c082e10aa1..179d92ad00 100644 --- a/src/mlpack/methods/lsh/lsh_main.cpp +++ b/src/mlpack/methods/lsh/lsh_main.cpp @@ -220,8 +220,11 @@ static void mlpackMain() if (trueNeighbors.n_rows != neighbors.n_rows || trueNeighbors.n_cols != neighbors.n_cols) { - Log::Fatal << "The true neighbors file must have the same number of " - << "values as the set of neighbors being queried!" << endl; + // Delete the model if needed. + if (CLI::HasParam("reference")) + delete allkann; + Log::Fatal << "The true neighbors file must have the same number of " + << "values as the set of neighbors being queried!" << endl; } Log::Info << "Using true neighbor indices from '" diff --git a/src/mlpack/methods/neighbor_search/kfn_main.cpp b/src/mlpack/methods/neighbor_search/kfn_main.cpp index d046655888..26485bbe77 100644 --- a/src/mlpack/methods/neighbor_search/kfn_main.cpp +++ b/src/mlpack/methods/neighbor_search/kfn_main.cpp @@ -188,8 +188,6 @@ static void mlpackMain() if (CLI::HasParam("reference")) { - kfn = new KFNModel(); - // Get all the parameters. RequireParamInSet("tree_type", { "kd", "cover", "r", "r-star", "ball", "x", "hilbert-r", "r-plus", "r-plus-plus", "vp", "rp", "max-rp", @@ -197,6 +195,8 @@ static void mlpackMain() const string treeType = CLI::GetParam("tree_type"); const bool randomBasis = CLI::HasParam("random_basis"); + kfn = new KFNModel(); + KFNModel::TreeTypes tree = KFNModel::KD_TREE; if (treeType == "kd") tree = KFNModel::KD_TREE; @@ -274,8 +274,12 @@ static void mlpackMain() << queryData.n_rows << "x" << queryData.n_cols << ")." << endl; if (queryData.n_rows != kfn->Dataset().n_rows) { + // Clean memory if needed. + const size_t dimensions = kfn->Dataset().n_rows; + if (CLI::HasParam("reference")) + delete kfn; Log::Fatal << "Query has invalid dimensions (" << queryData.n_rows << - "); should be " << kfn->Dataset().n_rows << "!" << endl; + "); should be " << dimensions << "!" << endl; } } @@ -284,18 +288,26 @@ static void mlpackMain() // we only test the upper bound. if (k > kfn->Dataset().n_cols) { + // Clean memory if needed. + const size_t referencePoints = kfn->Dataset().n_cols; + if (CLI::HasParam("reference")) + delete kfn; Log::Fatal << "Invalid k: " << k << "; must be greater than 0 and less " << "than or equal to the number of reference points (" - << kfn->Dataset().n_cols << ")." << endl; + << referencePoints << ")." << endl; } // Sanity check on k value: must not be equal to the number of reference // points when query data has not been provided. if (!CLI::HasParam("query") && k == kfn->Dataset().n_cols) { + // Clean memory if needed. + const size_t referencePoints = kfn->Dataset().n_cols; + if (CLI::HasParam("reference")) + delete kfn; Log::Fatal << "Invalid k: " << k << "; must be less than the number of " - << "reference points (" << kfn->Dataset().n_cols << ") " - << "if query data has not been provided." << endl; + << "reference points (" << referencePoints << ") if query data has " + << "not been provided." << endl; } // Now run the search. @@ -321,8 +333,13 @@ static void mlpackMain() if (trueDistances.n_rows != distances.n_rows || trueDistances.n_cols != distances.n_cols) + { + // Clean memory if needed. + if (CLI::HasParam("reference")) + delete kfn; Log::Fatal << "The true distances file must have the same number of " << "values than the set of distances being queried!" << endl; + } Log::Info << "Effective error: " << KFN::EffectiveError(distances, trueDistances) << endl; @@ -341,8 +358,13 @@ static void mlpackMain() if (trueNeighbors.n_rows != neighbors.n_rows || trueNeighbors.n_cols != neighbors.n_cols) + { + // Clean memory if needed. + if (CLI::HasParam("reference")) + delete kfn; Log::Fatal << "The true neighbors file must have the same number of " << "values than the set of neighbors being queried!" << endl; + } Log::Info << "Recall: " << KFN::Recall(neighbors, trueNeighbors) << endl; } diff --git a/src/mlpack/methods/neighbor_search/knn_main.cpp b/src/mlpack/methods/neighbor_search/knn_main.cpp index ece34d861b..01918ec1b0 100644 --- a/src/mlpack/methods/neighbor_search/knn_main.cpp +++ b/src/mlpack/methods/neighbor_search/knn_main.cpp @@ -200,8 +200,6 @@ static void mlpackMain() if (CLI::HasParam("reference")) { - knn = new KNNModel(); - // Get all the parameters. const string treeType = CLI::GetParam("tree_type"); const bool randomBasis = CLI::HasParam("random_basis"); @@ -210,6 +208,9 @@ static void mlpackMain() RequireParamInSet("tree_type", { "kd", "cover", "r", "r-star", "ball", "x", "hilbert-r", "r-plus", "r-plus-plus", "spill", "vp", "rp", "max-rp", "ub", "oct" }, true, "unknown tree type"); + + knn = new KNNModel(); + if (treeType == "kd") tree = KNNModel::KD_TREE; else if (treeType == "cover") @@ -292,8 +293,12 @@ static void mlpackMain() << queryData.n_rows << "x" << queryData.n_cols << ")." << endl; if (queryData.n_rows != knn->Dataset().n_rows) { + // Clean memory if needed before crashing. + const size_t dimensions = knn->Dataset().n_rows; + if (CLI::HasParam("reference")) + delete knn; Log::Fatal << "Query has invalid dimensions(" << queryData.n_rows << - "); should be " << knn->Dataset().n_rows << "!" << endl; + "); should be " << dimensions << "!" << endl; } } @@ -302,18 +307,26 @@ static void mlpackMain() // we only test the upper bound. if (k > knn->Dataset().n_cols) { + // Clean memory if needed before crashing. + const size_t referencePoints = knn->Dataset().n_cols; + if (CLI::HasParam("reference")) + delete knn; Log::Fatal << "Invalid k: " << k << "; must be greater than 0 and less " << "than or equal to the number of reference points (" - << knn->Dataset().n_cols << ")." << endl; + << referencePoints << ")." << endl; } // Sanity check on k value: must not be equal to the number of reference // points when query data has not been provided. if (!CLI::HasParam("query") && k == knn->Dataset().n_cols) { + // Clean memory if needed before crashing. + const size_t referencePoints = knn->Dataset().n_cols; + if (CLI::HasParam("reference")) + delete knn; Log::Fatal << "Invalid k: " << k << "; must be less than the number of " - << "reference points (" << knn->Dataset().n_cols << ") " - << "if query data has not been provided." << endl; + << "reference points (" << referencePoints << ") if query data has " + << "not been provided." << endl; } // Now run the search. @@ -339,8 +352,12 @@ static void mlpackMain() if (trueDistances.n_rows != distances.n_rows || trueDistances.n_cols != distances.n_cols) + { + if (CLI::HasParam("reference")) + delete knn; Log::Fatal << "The true distances file must have the same number of " << "values than the set of distances being queried!" << endl; + } Log::Info << "Effective error: " << KNN::EffectiveError(distances, trueDistances) << endl; @@ -359,8 +376,12 @@ static void mlpackMain() if (trueNeighbors.n_rows != neighbors.n_rows || trueNeighbors.n_cols != neighbors.n_cols) + { + if (CLI::HasParam("reference")) + delete knn; Log::Fatal << "The true neighbors file must have the same number of " << "values than the set of neighbors being queried!" << endl; + } Log::Info << "Recall: " << KNN::Recall(neighbors, trueNeighbors) << endl; } diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index 2410d2b1cf..d1b1dd5816 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -40,4 +40,12 @@ add_markdown_docs(preprocess_imputer "cli" "preprocessing") add_cli_executable(preprocess_scale) add_python_binding(preprocess_scale) -add_markdown_docs(preprocess_scale "cli;python" "preprocessing") +add_julia_binding(preprocess_scale) +add_markdown_docs(preprocess_scale "cli;python;julia" "preprocessing") + +if (STB_AVAILABLE) + add_cli_executable(image_converter) + add_python_binding(image_converter) + add_julia_binding(image_converter) + add_markdown_docs(image_converter "cli;python;julia" "preprocessing") +endif () \ No newline at end of file diff --git a/src/mlpack/methods/preprocess/image_converter_main.cpp b/src/mlpack/methods/preprocess/image_converter_main.cpp new file mode 100644 index 0000000000..0d4beecf4b --- /dev/null +++ b/src/mlpack/methods/preprocess/image_converter_main.cpp @@ -0,0 +1,114 @@ +/** + * @file image_converter_main.cpp + * @author Jeffin Sam + * + * A CLI executable to load and save a image dataset. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include + +using namespace mlpack; +using namespace mlpack::util; +using namespace arma; +using namespace std; +using namespace mlpack::data; + +PROGRAM_INFO("Image Converter", + // Short description. + "A utility to load an image or set of images into a single dataset that" + " can then be used by other mlpack methods and utilities. This can also" + " unpack an image dataset into individual files, for instance after mlpack" + " methods have been used.", + // Long description. + "This utility takes an image or an array of images and loads them to a" + " matrix. You can optionally specify the height " + + PRINT_PARAM_STRING("height") + " width " + PRINT_PARAM_STRING("width") + + " and channel " + PRINT_PARAM_STRING("channels") + " of the images that" + " needs to be loaded; otherwise, these parameters will be automatically" + " detected from the image." + "\n" + "There are other options too, that can be specified such as " + + PRINT_PARAM_STRING("quality") + + ".\n\n" + + "You can also provide a dataset and save them as images using " + + PRINT_PARAM_STRING("dataset") + " and " + PRINT_PARAM_STRING("save") + + " as an parameter. An example to load an image : " + + "\n\n" + + PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, + "channels", 3, "output", "Y") + + "\n\n" + + " An example to save an image is :" + + "\n\n" + + PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, + "channels", 3, "dataset", "Y", "save", true), + SEE_ALSO("@preprocess_binarize", "#preprocess_binarize"), + SEE_ALSO("@preprocess_describe", "#preprocess_describe"), + SEE_ALSO("@preprocess_imputer", "#preprocess_imputer")); + +// DEFINE PARAM +PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which have to " + "be loaded/saved.", "i"); + +PARAM_INT_IN("width", "Width of the image.", "w", 0); +PARAM_INT_IN("channels", "Number of channels in the image.", "c", 0); + +PARAM_MATRIX_OUT("output", "Matrix to save images data to, Only" + "needed if you are specifying 'save' option.", "o"); + +PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", + "q", 90); + +PARAM_INT_IN("height", "Height of the images.", "H", 0); +PARAM_FLAG("save", "Save a dataset as images.", "s"); +PARAM_MATRIX_IN("dataset", "Input matrix to save as images.", "I"); + +static void mlpackMain() +{ + Timer::Start("Loading/Saving Image"); + // Parse command line options. + const vector fileNames = CLI::GetParam >("input"); + arma::mat out; + + if (!CLI::HasParam("save")) + { + ReportIgnoredParam("width", "Width of image is determined from file."); + ReportIgnoredParam("height", "Height of image is determined from file."); + ReportIgnoredParam("channels", "Number of channels determined from file."); + data::ImageInfo info; + Load(fileNames, out, info, true); + if (CLI::HasParam("output")) + CLI::GetParam("output") = std::move(out); + } + else + { + RequireNoneOrAllPassed({ "save", "width", "height", "channels", "dataset" } + , true, "Image size information is needed when 'save' is specified!"); + // Positive value for width. + RequireParamValue("width", [](int x) { return x >= 0;}, true, + "width must be positive"); + // Positive value for height. + RequireParamValue("height", [](int x) { return x >= 0;}, true, + "height must be positive"); + // Positive value for channel. + RequireParamValue("channels", [](int x) { return x >= 0;}, true, + "channels must be positive"); + // Positive value for quality. + RequireParamValue("quality", [](int x) { return x >= 0;}, true, + "quality must be positive"); + + const size_t height = CLI::GetParam("height"); + const size_t width = CLI::GetParam("width"); + const size_t channels = CLI::GetParam("channels"); + const size_t quality = CLI::GetParam("quality"); + data::ImageInfo info(width, height, channels, quality); + Save(fileNames, CLI::GetParam("dataset"), info, true); + } +} + diff --git a/src/mlpack/methods/preprocess/preprocess_scale_main.cpp b/src/mlpack/methods/preprocess/preprocess_scale_main.cpp index bcbdc46f08..f4694655e0 100644 --- a/src/mlpack/methods/preprocess/preprocess_scale_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_scale_main.cpp @@ -10,9 +10,10 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include +#include #include #include -#include +#include #include #include #include @@ -131,6 +132,7 @@ static void mlpackMain() { m = new ScalingModel(CLI::GetParam("min_value"), CLI::GetParam("max_value"), CLI::GetParam("epsilon")); + if (scalerMethod == "standard_scaler") { m->ScalerType() = ScalingModel::ScalerTypes::STANDARD_SCALER; @@ -155,8 +157,20 @@ static void mlpackMain() { m->ScalerType() = ScalingModel::ScalerTypes::PCA_WHITENING; } - m->Fit(input); + + // Fit() can throw an exception on invalid inputs, so we have to catch that + // and clean the memory in that situation. + try + { + m->Fit(input); + } + catch (std::exception& e) + { + delete m; + throw; + } } + if (!CLI::HasParam("inverse_scaling")) { m->Transform(input, output); @@ -165,8 +179,8 @@ static void mlpackMain() { if (!CLI::HasParam("input_model")) { - delete(m); - throw std::runtime_error("Please provide a saved model"); + delete m; + throw std::runtime_error("Please provide a saved model."); } m->InverseTransform(input, output); } @@ -175,6 +189,6 @@ static void mlpackMain() if (CLI::HasParam("output")) CLI::GetParam("output") = std::move(output); Timer::Stop("feature_scaling"); - if (CLI::HasParam("output_model")) - CLI::GetParam("output_model") = m; + + CLI::GetParam("output_model") = m; } diff --git a/src/mlpack/methods/range_search/range_search_main.cpp b/src/mlpack/methods/range_search/range_search_main.cpp index 346e133c05..26f2ec2396 100644 --- a/src/mlpack/methods/range_search/range_search_main.cpp +++ b/src/mlpack/methods/range_search/range_search_main.cpp @@ -155,8 +155,6 @@ static void mlpackMain() const bool singleMode = CLI::HasParam("single_mode"); if (CLI::HasParam("reference")) { - rs = new RSModel(); - // Get all the parameters. const string treeType = CLI::GetParam("tree_type"); RequireParamInSet("tree_type", { "kd", "cover", "r", "r-star", @@ -164,6 +162,8 @@ static void mlpackMain() "ub", "oct" }, true, "unknown tree type"); const bool randomBasis = CLI::HasParam("random_basis"); + rs = new RSModel(); + RSModel::TreeTypes tree = RSModel::KD_TREE; if (treeType == "kd") tree = RSModel::KD_TREE; diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index 62bb0633fc..b61cc561e7 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -169,10 +169,10 @@ class Acrobot nextState.Theta2() = Wrap(currentNextState[1], -M_PI, M_PI); //! The value of angular velocity is bounded in min and max value. - nextState.AngularVelocity1() = std::min( - std::max(currentNextState[2], -maxVel1), maxVel1); - nextState.AngularVelocity2() = std::min( - std::max(currentNextState[3], -maxVel2), maxVel2); + nextState.AngularVelocity1() = math::ClampRange(currentNextState[2], + -maxVel1, maxVel1); + nextState.AngularVelocity2() = math::ClampRange(currentNextState[3], + -maxVel2, maxVel2); // Check if the episode has terminated. bool done = IsTerminal(nextState); diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 5837e1b147..c1336428a6 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -18,6 +18,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_MOUNTAIN_CAR_HPP #include +#include namespace mlpack { namespace rl { @@ -96,7 +97,7 @@ class ContinuousMountainCar * @param positionGoal Final target position. * @param velocityMin Minimum legal velocity. * @param velocityMax Maximum legal velocity. - * @param power Power generated by car. + * @param duration Time Duration for which force is applied on the car. * @param doneReward Reward recieved by the agent on success. * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. @@ -106,7 +107,7 @@ class ContinuousMountainCar const double positionGoal = 0.45, const double velocityMin = -0.07, const double velocityMax = 0.07, - const double power = 0.0015, + const double duration = 0.0015, const double doneReward = 100, const size_t maxSteps = 0) : positionMin(positionMin), @@ -114,7 +115,7 @@ class ContinuousMountainCar positionGoal(positionGoal), velocityMin(velocityMin), velocityMax(velocityMax), - power(power), + duration(duration), doneReward(doneReward), maxSteps(maxSteps), stepsPerformed(0) @@ -136,16 +137,16 @@ class ContinuousMountainCar stepsPerformed++; // Calculate acceleration. - double force = std::min(std::max(action.action[0], -1.0), 1.0); + double force = math::ClampRange(action.action[0], -1.0, 1.0); // Update states. - nextState.Velocity() = state.Velocity() + force * power - 0.0025 * + nextState.Velocity() = state.Velocity() + force * duration - 0.0025 * std::cos(3 * state.Position()); - nextState.Velocity() = std::min( - std::max(nextState.Velocity(), velocityMin), velocityMax); + nextState.Velocity() = math::ClampRange(nextState.Velocity(), + velocityMin, velocityMax); nextState.Position() = state.Position() + nextState.Velocity(); - nextState.Position() = std::min( - std::max(nextState.Position(), positionMin), positionMax); + nextState.Position() = math::ClampRange(nextState.Position(), + positionMin, positionMax); if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; @@ -236,8 +237,8 @@ class ContinuousMountainCar //! Locally-stored maximum legal velocity. double velocityMax; - //! Locally-stored power. - double power; + //! Locally-stored duration. + double duration; //! Locally-stored done reward. double doneReward; diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index b02b631cd4..a83e44d4fc 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -17,6 +17,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_MOUNTAIN_CAR_HPP #include +#include namespace mlpack { namespace rl { @@ -134,13 +135,13 @@ class MountainCar int direction = action - 1; nextState.Velocity() = state.Velocity() + 0.001 * direction - 0.0025 * std::cos(3 * state.Position()); - nextState.Velocity() = std::min( - std::max(nextState.Velocity(), velocityMin), velocityMax); + nextState.Velocity() = math::ClampRange(nextState.Velocity(), + velocityMin, velocityMax); // Update states. nextState.Position() = state.Position() + nextState.Velocity(); - nextState.Position() = std::min( - std::max(nextState.Position(), positionMin), positionMax); + nextState.Position() = math::ClampRange(nextState.Position(), + positionMin, positionMax); if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index e7133bdf6b..55c02668cd 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -17,6 +17,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP #include +#include namespace mlpack { namespace rl { @@ -140,8 +141,7 @@ class Pendulum const double length = 1.0; // Get action and clip the values between max and min limits. - double torque = std::min( - std::max(action.action[0], -maxTorque), maxTorque); + double torque = math::ClampRange(action.action[0], -maxTorque, maxTorque); // Calculate costs of taking this action in the current state. double costs = std::pow(AngleNormalize(theta), 2) + 0.1 * @@ -151,8 +151,8 @@ class Pendulum double newAngularVelocity = angularVelocity + (-3.0 * gravity / (2 * length) * std::sin(theta + M_PI) + 3.0 / std::pow(mass * length, 2) * torque) * dt; - nextState.AngularVelocity() = std::min(std::max(newAngularVelocity, - -maxAngularVelocity), maxAngularVelocity); + nextState.AngularVelocity() = math::ClampRange(newAngularVelocity, + -maxAngularVelocity, maxAngularVelocity); nextState.Theta() = theta + newAngularVelocity * dt; // Check if the episode has terminated diff --git a/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp b/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp index a7ce90038e..765519c89e 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp @@ -13,6 +13,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_REWARD_CLIPPING_HPP #include +#include namespace mlpack { namespace rl { @@ -91,7 +92,7 @@ class RewardClipping // Get original unclipped reward from base environment. double unclippedReward = environment.Sample(state, action, nextState); // Clip rewards according to the min and max limit and return. - return std::min(std::max(unclippedReward, minReward), maxReward); + return math::ClampRange(unclippedReward, minReward, maxReward); } /** diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index d90b738fcf..38234ed673 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -179,6 +179,7 @@ double QLearning< // Compute the update target. arma::mat target; learningNetwork.Forward(sampledStates, target); + /** * If the agent is at a terminal state, then we don't need to add the * discounted reward. At terminal state, the agent wont perform any @@ -199,7 +200,7 @@ double QLearning< // Learn from experience. arma::mat gradients; - learningNetwork.Backward(target, gradients); + learningNetwork.Backward(sampledStates, target, gradients); replayMethod.Update(target, sampledActions, nextActionValues, gradients); diff --git a/src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp b/src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp index 1ca53315bb..b532eab03b 100644 --- a/src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp +++ b/src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp @@ -301,12 +301,13 @@ class NStepQLearningWorker target = config.Discount() * target + std::get<2>(transition); // Compute the training target for current state. - network.Forward(std::get<0>(transition).Encode(), actionValue); + arma::mat input = std::get<0>(transition).Encode(); + network.Forward(input, actionValue); actionValue[std::get<1>(transition)] = target; // Compute gradient. arma::mat gradients; - network.Backward(actionValue, gradients); + network.Backward(input, actionValue, gradients); // Accumulate gradients. totalGradients += gradients; diff --git a/src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp b/src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp index 02a1c738f8..d7132bcc95 100644 --- a/src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp +++ b/src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp @@ -301,12 +301,13 @@ class OneStepQLearningWorker config.Discount() * targetActionValue; // Compute the training target for current state. - network.Forward(std::get<0>(transition).Encode(), actionValue); + arma::mat input = std::get<0>(transition).Encode(); + network.Forward(input, actionValue); actionValue[std::get<1>(transition)] = targetActionValue; // Compute gradient. arma::mat gradients; - network.Backward(actionValue, gradients); + network.Backward(input, actionValue, gradients); // Accumulate gradients. totalGradients += gradients; diff --git a/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp b/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp index ef974f0525..6dca26c194 100644 --- a/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp +++ b/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp @@ -314,12 +314,13 @@ class OneStepSarsaWorker config.Discount() * targetActionValue; // Compute the training target for current state. - network.Forward(std::get<0>(transition).Encode(), actionValue); + arma::mat input = std::get<0>(transition).Encode(); + network.Forward(input, actionValue); actionValue[std::get<1>(transition)] = targetActionValue; // Compute gradient. arma::mat gradients; - network.Backward(actionValue, gradients); + network.Backward(input, actionValue, gradients); // Accumulate gradients. totalGradients += gradients; diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index a1a865a1cc..77e4fcc2e4 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -137,6 +137,7 @@ add_executable(mlpack_test main_tests/kfn_test.cpp main_tests/knn_test.cpp main_tests/linear_regression_test.cpp + main_tests/image_converter_test.cpp main_tests/linear_svm_test.cpp main_tests/logistic_regression_test.cpp main_tests/local_coordinate_coding_test.cpp @@ -186,11 +187,11 @@ add_custom_command(TARGET mlpack_test ) add_custom_command(TARGET mlpack_test POST_BUILD - COMMAND ${CMAKE_COMMAND} -E tar xjpf mnist_first250_training_4s_and_9s.tar.bz2 - COMMAND ${CMAKE_COMMAND} -E tar xjpf digits_train.tar.bz2 - COMMAND ${CMAKE_COMMAND} -E tar xjpf digits_test.tar.bz2 - COMMAND ${CMAKE_COMMAND} -E tar xjpf digits_train_label.tar.bz2 - COMMAND ${CMAKE_COMMAND} -E tar xjpf digits_test_label.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf mnist_first250_training_4s_and_9s.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf digits_train.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf digits_test.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf digits_train_label.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf digits_test_label.tar.bz2 WORKING_DIRECTORY ${PROJECT_BINARY_DIR} ) diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 93f00a0ed9..78b32af158 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -133,7 +133,7 @@ void CheckHardTanHActivationCorrect(const arma::colvec input, // Test the activation function using the entire vector as input. arma::colvec activations; - htf.Forward(std::move(input), std::move(activations)); + htf.Forward(input, activations); for (size_t i = 0; i < activations.n_elem; i++) { BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3); @@ -157,7 +157,7 @@ void CheckHardTanHDerivativeCorrect(const arma::colvec input, // This error vector will be set to 1 to get the derivatives. arma::colvec error = arma::ones(input.n_elem); - htf.Backward(std::move(input), std::move(error), std::move(derivatives)); + htf.Backward(input, error, derivatives); for (size_t i = 0; i < derivatives.n_elem; i++) { @@ -179,7 +179,7 @@ void CheckLeakyReLUActivationCorrect(const arma::colvec input, // Test the activation function using the entire vector as input. arma::colvec activations; - lrf.Forward(std::move(input), std::move(activations)); + lrf.Forward(input, activations); for (size_t i = 0; i < activations.n_elem; i++) { BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3); @@ -204,7 +204,7 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, // This error vector will be set to 1 to get the derivatives. arma::colvec error = arma::ones(input.n_elem); - lrf.Backward(std::move(input), std::move(error), std::move(derivatives)); + lrf.Backward(input, error, derivatives); for (size_t i = 0; i < derivatives.n_elem; i++) { BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3); @@ -219,14 +219,14 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, * @param target Target data used to evaluate the ELU activation. */ void CheckELUActivationCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { // Initialize ELU object with alpha = 1.0. ELU<> lrf(1.0); // Test the activation function using the entire vector as input. arma::colvec activations; - lrf.Forward(std::move(input), std::move(activations)); + lrf.Forward(input, activations); for (size_t i = 0; i < activations.n_elem; i++) { BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3); @@ -241,7 +241,7 @@ void CheckELUActivationCorrect(const arma::colvec input, * @param target Target data used to evaluate the ELU activation. */ void CheckELUDerivativeCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { // Initialize ELU object with alpha = 1.0. ELU<> lrf(1.0); @@ -251,9 +251,8 @@ void CheckELUDerivativeCorrect(const arma::colvec input, // This error vector will be set to 1 to get the derivatives. arma::colvec error = arma::ones(input.n_elem); - lrf.Forward(std::move(input), std::move(activations)); - lrf.Backward(std::move(activations), std::move(error), - std::move(derivatives)); + lrf.Forward(input, activations); + lrf.Backward(activations, error, derivatives); for (size_t i = 0; i < derivatives.n_elem; i++) { BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3); @@ -262,20 +261,20 @@ void CheckELUDerivativeCorrect(const arma::colvec input, /* * Implementation of the PReLU activation function test. The function - * is implemented as PReLU layer in the file perametric_relu.hpp + * is implemented as PReLU layer in the file parametric_relu.hpp. * * @param input Input data used for evaluating the PReLU activation * function. * @param target Target data used to evaluate the PReLU activation. */ void CheckPReLUActivationCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { PReLU<> prelu; // Test the activation function using the entire vector as input. arma::colvec activations; - prelu.Forward(std::move(input), std::move(activations)); + prelu.Forward(input, activations); for (size_t i = 0; i < activations.n_elem; i++) { BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3); @@ -285,14 +284,14 @@ void CheckPReLUActivationCorrect(const arma::colvec input, /* * Implementation of the PReLU activation function derivative test. * The function is implemented as PReLU layer in the file - * perametric_relu.hpp + * parametric_relu.hpp * * @param input Input data used for evaluating the PReLU activation * function. * @param target Target data used to evaluate the PReLU activation. */ void CheckPReLUDerivativeCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { PReLU<> prelu; @@ -301,7 +300,7 @@ void CheckPReLUDerivativeCorrect(const arma::colvec input, // This error vector will be set to 1 to get the derivatives. arma::colvec error = arma::ones(input.n_elem); - prelu.Backward(std::move(input), std::move(error), std::move(derivatives)); + prelu.Backward(input, error, derivatives); for (size_t i = 0; i < derivatives.n_elem; i++) { BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3); @@ -311,14 +310,14 @@ void CheckPReLUDerivativeCorrect(const arma::colvec input, /* * Implementation of the PReLU activation function gradient test. * The function is implemented as PReLU layer in the file - * perametric_relu.hpp + * parametric_relu.hpp * * @param input Input data used for evaluating the PReLU activation * function. * @param target Target data used to evaluate the PReLU gradient. */ void CheckPReLUGradientCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { PReLU<> prelu; @@ -327,7 +326,7 @@ void CheckPReLUGradientCorrect(const arma::colvec input, // This error vector will be set to 1 to get the gradient. arma::colvec error = arma::ones(input.n_elem); - prelu.Gradient(std::move(input), std::move(error), std::move(gradient)); + prelu.Gradient(input, error, gradient); BOOST_REQUIRE_EQUAL(gradient.n_rows, 1); BOOST_REQUIRE_EQUAL(gradient.n_cols, 1); BOOST_REQUIRE_CLOSE(gradient(0), target(0), 1e-3); @@ -341,13 +340,13 @@ void CheckPReLUGradientCorrect(const arma::colvec input, * @param target Target data used to evaluate the Hard Shrink activation. */ void CheckHardShrinkActivationCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { HardShrink<> hardshrink; // Test the activation function using the entire vector as input. arma::colvec activations; - hardshrink.Forward(std::move(input), std::move(activations)); + hardshrink.Forward(input, activations); for (size_t i = 0; i < activations.n_elem; i++) { BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3); @@ -363,7 +362,7 @@ void CheckHardShrinkActivationCorrect(const arma::colvec input, * @param target Target data used to evaluate the HardShrink activation. */ void CheckHardShrinkDerivativeCorrect(const arma::colvec input, - const arma::colvec target) + const arma::colvec target) { HardShrink<> hardshrink; @@ -372,8 +371,53 @@ void CheckHardShrinkDerivativeCorrect(const arma::colvec input, // This error vector will be set to 1 to get the derivatives. arma::colvec error = arma::ones(input.n_elem); - hardshrink.Backward(std::move(input), std::move(error), std::move( - derivatives)); + hardshrink.Backward(input, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; i++) + { + BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3); + } +} + +/* + * Implementation of the Soft Shrink activation function test. The function is + * implemented as Soft Shrink layer in the file softshrink.hpp. + * + * @param input Input data used for evaluating the Soft Shrink activation function. + * @param target Target data used to evaluate the Soft Shrink activation. + */ +void CheckSoftShrinkActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + SoftShrink<> softshrink; + + // Test the activation function using the entire vector as input. + arma::colvec activations; + softshrink.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; i++) + { + BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3); + } +} + +/* + * Implementation of the Soft Shrink activation function derivative test. + * The derivative function is implemented as Soft Shrink layer in the file + * softshrink.hpp + * + * @param input Input data used for evaluating the Soft Shrink activation function. + * @param target Target data used to evaluate the Soft Shrink activation. + */ +void CheckSoftShrinkDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + SoftShrink<> softshrink; + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + softshrink.Backward(input, error, derivatives); for (size_t i = 0; i < derivatives.n_elem; i++) { BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3); @@ -392,7 +436,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) SELU selu; - selu.Forward(std::move(input), output); + selu.Forward(input, output); BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); @@ -414,7 +458,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest) SELU selu; - selu.Forward(std::move(input), output); + selu.Forward(input, output); BOOST_REQUIRE_GE(arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); @@ -438,23 +482,69 @@ BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) SELU selu; - selu.Forward(std::move(input), activations); - selu.Backward(std::move(activations), std::move(error), - std::move(derivatives)); + selu.Forward(input, activations); + selu.Backward(activations, error, derivatives); BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - selu.Lambda())), 10e-4); input.fill(-1); - selu.Forward(std::move(input), activations); - selu.Backward(std::move(activations), std::move(error), - std::move(derivatives)); + selu.Forward(input, activations); + selu.Backward(activations, error, derivatives); BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - selu.Lambda() * selu.Alpha() - arma::mean(activations))), 10e-4); } +/** + * Implementation of the CELU activation function test. The function is + * implemented as CELU layer in the file celu.hpp. + * + * @param input Input data used for evaluating the CELU activation function. + * @param target Target data used to evaluate the CELU activation. + */ +void CheckCELUActivationCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize CELU object with alpha = 1.0. + CELU<> lrf(1.0); + + // Test the activation function using the entire vector as input. + arma::colvec activations; + lrf.Forward(input, activations); + for (size_t i = 0; i < activations.n_elem; i++) + { + BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3); + } +} + +/** + * Implementation of the CELU activation function derivative test. The function + * is implemented as CELU layer in the file celu.hpp. + * + * @param input Input data used for evaluating the CELU activation function. + * @param target Target data used to evaluate the CELU activation. + */ +void CheckCELUDerivativeCorrect(const arma::colvec input, + const arma::colvec target) +{ + // Initialize CELU object with alpha = 1.0. + CELU<> lrf(1.0); + + // Test the calculation of the derivatives using the entire vector as input. + arma::colvec derivatives, activations; + + // This error vector will be set to 1 to get the derivatives. + arma::colvec error = arma::ones(input.n_elem); + lrf.Forward(input, activations); + lrf.Backward(activations, error, derivatives); + for (size_t i = 0; i < derivatives.n_elem; i++) + { + BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3); + } +} + /** * Basic test of the tanh function. */ @@ -629,12 +719,11 @@ BOOST_AUTO_TEST_CASE(CReLUFunctionTest) CReLU<> crelu; // Test the activation function using the entire vector as input. arma::colvec activations; - crelu.Forward(std::move(activationData), std::move(activations)); + crelu.Forward(activationData, activations); arma::colvec derivatives; // This error vector will be set to 1 to get the derivatives. arma::colvec error = arma::ones(desiredActivations.n_elem); - crelu.Backward(std::move(desiredActivations), std::move(error), - std::move(derivatives)); + crelu.Backward(desiredActivations, error, derivatives); for (size_t i = 0; i < activations.n_elem; i++) { BOOST_REQUIRE_CLOSE(activations.at(i), desiredActivations.at(i), 1e-3); @@ -772,4 +861,35 @@ BOOST_AUTO_TEST_CASE(HardShrinkFunctionTest) desiredDerivatives); } +/** + * Basic test of the Soft Shrink function. + */ +BOOST_AUTO_TEST_CASE(SoftShrinkFunctionTest) +{ + const arma::colvec desiredActivations("-1.5 2.7 4 -99.7 0.5 -0.5 1.5 0"); + + const arma::colvec desiredDerivatives("1 1 1 1 1 1 1 0"); + + CheckSoftShrinkActivationCorrect(activationData, + desiredActivations); + CheckSoftShrinkDerivativeCorrect(desiredActivations, + desiredDerivatives); +} + +/** + * Basic test of the CELU activation function. + */ +BOOST_AUTO_TEST_CASE(CELUFunctionTest) +{ + const arma::colvec desiredActivations("-0.86466472 3.2 4.5 \ + -1 1 -0.63212056 2 0"); + + const arma::colvec desiredDerivatives("0.42119275 1 1 \ + 0.36787944 1 \ + 0.5314636 1 1"); + + CheckCELUActivationCorrect(activationData, desiredActivations); + CheckCELUDerivativeCorrect(desiredActivations, desiredDerivatives); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp index 86d74034c4..0065877abe 100644 --- a/src/mlpack/tests/adaboost_test.cpp +++ b/src/mlpack/tests/adaboost_test.cpp @@ -58,10 +58,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundIris) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double hammingLoss = (double) countError / labels.n_cols; // Check that ztProduct is finite. @@ -96,10 +93,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris) Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != perceptronPrediction(i)) - countWeakLearnerError++; + size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. @@ -110,10 +104,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels);; double error = (double) countError / labels.n_cols; BOOST_REQUIRE_LE(error, weakLearnerErrorRate); @@ -151,10 +142,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double hammingLoss = (double) countError / labels.n_cols; // Check that ztProduct is finite. @@ -187,10 +175,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn) Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != perceptronPrediction(i)) - countWeakLearnerError++; + size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. @@ -201,10 +186,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double error = (double) countError / labels.n_cols; BOOST_REQUIRE_LE(error, weakLearnerErrorRate); @@ -242,10 +224,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels == predictedLabels); double hammingLoss = (double) countError / labels.n_cols; // Check that ztProduct is finite. @@ -278,10 +257,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData) Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != perceptronPrediction(i)) - countWeakLearnerError++; + size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. @@ -292,10 +268,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double error = (double) countError / labels.n_cols; BOOST_REQUIRE_LE(error, weakLearnerErrorRate); @@ -332,10 +305,7 @@ BOOST_AUTO_TEST_CASE(HammingLossIris_DS) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double hammingLoss = (double) countError / labels.n_cols; // Check that ztProduct is finite. @@ -371,10 +341,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS) ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != dsPrediction(i)) - countWeakLearnerError++; + size_t countWeakLearnerError = arma::accu(labels != dsPrediction); double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. @@ -387,10 +354,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double error = (double) countError / labels.n_cols; BOOST_REQUIRE_LE(error, weakLearnerErrorRate); @@ -430,10 +394,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double hammingLoss = (double) countError / labels.n_cols; // Check that ztProduct is finite. @@ -466,11 +427,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS) ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != dsPrediction(i)) - countWeakLearnerError++; - + size_t countWeakLearnerError = arma::accu(labels != dsPrediction); double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. @@ -482,10 +439,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double error = (double) countError / labels.n_cols; BOOST_REQUIRE_LE(error, weakLearnerErrorRate); @@ -524,10 +478,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double hammingLoss = (double) countError / labels.n_cols; // Check that ztProduct is finite. @@ -561,10 +512,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS) ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != dsPrediction(i)) - countWeakLearnerError++; + size_t countWeakLearnerError = arma::accu(labels != dsPrediction); double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. @@ -577,10 +525,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS) arma::Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = 0; - for (size_t i = 0; i < labels.n_cols; i++) - if (labels(i) != predictedLabels(i)) - countError++; + size_t countError = arma::accu(labels != predictedLabels); double error = (double) countError / labels.n_cols; BOOST_REQUIRE_LE(error, weakLearnerErrorRate); @@ -650,11 +595,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_VERTEBRALCOL) BOOST_REQUIRE_CLOSE(arma::accu(probabilities.col(i)), 1, 1e-5); } - size_t localError = 0; - for (size_t i = 0; i < trueTestLabels.n_cols; i++) - if (trueTestLabels(i) != predictedLabels1(i)) - localError++; - + size_t localError = arma::accu(trueTestLabels != predictedLabels1); double lError = (double) localError / trueTestLabels.n_cols; BOOST_REQUIRE_LE(lError, 0.30); } @@ -722,11 +663,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP) BOOST_REQUIRE_CLOSE(arma::accu(probabilities.col(i)), 1, 1e-5); } - size_t localError = 0; - for (size_t i = 0; i < trueTestLabels.n_cols; i++) - if (trueTestLabels(i) != predictedLabels1(i)) - localError++; - + size_t localError = arma::accu(trueTestLabels != predictedLabels1); double lError = (double) localError / trueTestLabels.n_cols; BOOST_REQUIRE_LE(lError, 0.30); } @@ -793,10 +730,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_IRIS) BOOST_REQUIRE_CLOSE(arma::accu(probabilities.col(i)), 1, 1e-5); } - size_t localError = 0; - for (size_t i = 0; i < trueTestLabels.n_cols; i++) - if (trueTestLabels(i) != predictedLabels1(i)) - localError++; + size_t localError = arma::accu(trueTestLabels != predictedLabels1); double lError = (double) localError / labels.n_cols; BOOST_REQUIRE_LE(lError, 0.30); } @@ -851,11 +785,7 @@ BOOST_AUTO_TEST_CASE(TrainTest) arma::Row predictedLabels(testData.n_cols); a.Classify(testData, predictedLabels); - int localError = 0; - for (size_t i = 0; i < trueTestLabels.n_cols; i++) - if (trueTestLabels(i) != predictedLabels(i)) - localError++; - + int localError = arma::accu(trueTestLabels != predictedLabels); double lError = (double) localError / trueTestLabels.n_cols; BOOST_REQUIRE_LE(lError, 0.30); diff --git a/src/mlpack/tests/ann_dist_test.cpp b/src/mlpack/tests/ann_dist_test.cpp index cf9baca90d..6c6dad1371 100644 --- a/src/mlpack/tests/ann_dist_test.cpp +++ b/src/mlpack/tests/ann_dist_test.cpp @@ -30,7 +30,7 @@ BOOST_AUTO_TEST_SUITE(ANNDistTest); BOOST_AUTO_TEST_CASE(SimpleBernoulliDistributionTest) { arma::mat param = arma::mat("1 1 0"); - BernoulliDistribution<> module(std::move(param), false); + BernoulliDistribution<> module(param, false); arma::mat sample = module.Sample(); // As the probabilities are [1, 1, 0], the bernoulli samples should be @@ -53,7 +53,7 @@ BOOST_AUTO_TEST_CASE(JacobianBernoulliDistributionTest) arma::mat target; target.randn(targetElements, 1); - BernoulliDistribution<> module(std::move(param), false); + BernoulliDistribution<> module(param, false); const double perturbation = 1e-6; double outputA, outputB, original; @@ -66,16 +66,16 @@ BOOST_AUTO_TEST_CASE(JacobianBernoulliDistributionTest) { original = module.Probability()(j); module.Probability()(j) = original - perturbation; - outputA = module.LogProbability(std::move(target)); + outputA = module.LogProbability(target); module.Probability()(j) = original + perturbation; - outputB = module.LogProbability(std::move(target)); + outputB = module.LogProbability(target); module.Probability()(j) = original; outputB -= outputA; outputB /= 2 * perturbation; jacobianA(j) = outputB; } - module.LogProbBackward(std::move(target), std::move(jacobianB)); + module.LogProbBackward(target, jacobianB); BOOST_REQUIRE_LE(arma::max(arma::max(arma::abs(jacobianA - jacobianB))), 1e-5); } @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE(JacobianBernoulliDistributionLogisticTest) arma::mat target; target.randn(targetElements, 1); - BernoulliDistribution<> module(std::move(param)); + BernoulliDistribution<> module(param); const double perturbation = 1e-6; double outputA, outputB, original; @@ -110,10 +110,10 @@ BOOST_AUTO_TEST_CASE(JacobianBernoulliDistributionLogisticTest) original = module.Logits()(j); module.Logits()(j) = original - perturbation; LogisticFunction::Fn(module.Logits(), module.Probability()); - outputA = module.LogProbability(std::move(target)); + outputA = module.LogProbability(target); module.Logits()(j) = original + perturbation; LogisticFunction::Fn(module.Logits(), module.Probability()); - outputB = module.LogProbability(std::move(target)); + outputB = module.LogProbability(target); module.Logits()(j) = original; LogisticFunction::Fn(module.Logits(), module.Probability()); outputB -= outputA; @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(JacobianBernoulliDistributionLogisticTest) jacobianA(j) = outputB; } - module.LogProbBackward(std::move(target), std::move(jacobianB)); + module.LogProbBackward(target, jacobianB); BOOST_REQUIRE_LE(arma::max(arma::max(arma::abs(jacobianA - jacobianB))), 3e-5); } diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 136460c2f1..c064d61d9a 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -41,21 +41,21 @@ BOOST_AUTO_TEST_CASE(SimpleAddLayerTest) // Test the Forward function. input = arma::zeros(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(module.Parameters()), arma::accu(output)); // Test the Backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(delta)); // Test the forward function. input = arma::ones(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_CLOSE(10 + arma::accu(module.Parameters()), arma::accu(output), 1e-3); // Test the backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); BOOST_REQUIRE_CLOSE(arma::accu(output), arma::accu(delta), 1e-3); } @@ -131,20 +131,20 @@ BOOST_AUTO_TEST_CASE(SimpleConstantLayerTest) // Test the Forward function. input = arma::zeros(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 30.0); // Test the Backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); // Test the forward function. input = arma::ones(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 30.0); // Test the backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); } @@ -183,19 +183,19 @@ BOOST_AUTO_TEST_CASE(SimpleDropoutLayerTest) // Test the Forward function. arma::mat output; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_LE( arma::as_scalar(arma::abs(arma::mean(output) - (1 - p))), 0.05); // Test the Backward function. arma::mat delta; - module.Backward(std::move(input), std::move(input), std::move(delta)); + module.Backward(input, input, delta); BOOST_REQUIRE_LE( arma::as_scalar(arma::abs(arma::mean(delta) - (1 - p))), 0.05); // Test the Forward function. module.Deterministic() = true; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(input), arma::accu(output)); } @@ -219,7 +219,7 @@ BOOST_AUTO_TEST_CASE(DropoutProbabilityTest) module.Deterministic() = false; arma::mat output; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); // Return a column vector containing the indices of elements of X that // are non-zero, we just need the number of non-zero values. @@ -244,7 +244,7 @@ BOOST_AUTO_TEST_CASE(NoDropoutTest) module.Deterministic() = false; arma::mat output; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(input)); } @@ -267,7 +267,7 @@ BOOST_AUTO_TEST_CASE(SimpleAlphaDropoutLayerTest) // Test the Forward function when training phase. arma::mat output; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); // Check whether mean remains nearly same. BOOST_REQUIRE_LE( arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); @@ -278,13 +278,13 @@ BOOST_AUTO_TEST_CASE(SimpleAlphaDropoutLayerTest) // Test the Backward function when training phase. arma::mat delta; - module.Backward(std::move(input), std::move(input), std::move(delta)); + module.Backward(input, input, delta); BOOST_REQUIRE_LE( arma::as_scalar(arma::abs(arma::mean(delta) - 0)), 0.05); // Test the Forward function when testing phase. module.Deterministic() = true; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(input), arma::accu(output)); } @@ -308,7 +308,7 @@ BOOST_AUTO_TEST_CASE(AlphaDropoutProbabilityTest) module.Deterministic() = false; arma::mat output; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); // Return a column vector containing the indices of elements of X // that are not alphaDash, we just need the number of @@ -336,7 +336,7 @@ BOOST_AUTO_TEST_CASE(NoAlphaDropoutTest) module.Deterministic() = false; arma::mat output; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(input)); } @@ -353,13 +353,13 @@ BOOST_AUTO_TEST_CASE(SimpleLinearLayerTest) // Test the Forward function. input = arma::zeros(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_CLOSE(arma::accu( module.Parameters().submat(100, 0, module.Parameters().n_elem - 1, 0)), arma::accu(output), 1e-3); // Test the Backward function. - module.Backward(std::move(input), std::move(input), std::move(delta)); + module.Backward(input, input, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); } @@ -439,11 +439,11 @@ BOOST_AUTO_TEST_CASE(SimpleLinearNoBiasLayerTest) // Test the Forward function. input = arma::zeros(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(0, arma::accu(output)); // Test the Backward function. - module.Backward(std::move(input), std::move(input), std::move(delta)); + module.Backward(input, input, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); } @@ -457,13 +457,13 @@ BOOST_AUTO_TEST_CASE(SimplePaddingLayerTest) // Test the Forward function. input = arma::randu(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(input), arma::accu(output)); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows + 3); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols + 7); // Test the Backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); CheckMatrices(delta, input); } @@ -688,20 +688,20 @@ BOOST_AUTO_TEST_CASE(SimpleSelectLayerTest) // Test the Forward function. Select<> moduleA(3); - moduleA.Forward(std::move(input), std::move(outputA)); + moduleA.Forward(input, outputA); BOOST_REQUIRE_EQUAL(30, arma::accu(outputA)); // Test the Forward function. Select<> moduleB(3, 5); - moduleB.Forward(std::move(input), std::move(outputB)); + moduleB.Forward(input, outputB); BOOST_REQUIRE_EQUAL(15, arma::accu(outputB)); // Test the Backward function. - moduleA.Backward(std::move(input), std::move(outputA), std::move(delta)); + moduleA.Backward(input, outputA, delta); BOOST_REQUIRE_EQUAL(30, arma::accu(delta)); // Test the Backward function. - moduleB.Backward(std::move(input), std::move(outputA), std::move(delta)); + moduleB.Backward(input, outputA, delta); BOOST_REQUIRE_EQUAL(15, arma::accu(delta)); } @@ -715,14 +715,14 @@ BOOST_AUTO_TEST_CASE(SimpleJoinLayerTest) // Test the Forward function. Join<> module; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(50, arma::accu(output)); bool b = output.n_rows == 1 || output.n_cols == 1; BOOST_REQUIRE_EQUAL(b, true); // Test the Backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(50, arma::accu(delta)); b = delta.n_rows == input.n_rows && input.n_cols; @@ -744,18 +744,17 @@ BOOST_AUTO_TEST_CASE(SimpleAddMergeLayerTest) for (size_t m = 0; m < numMergeModules; ++m) { IdentityLayer<> identityLayer; - identityLayer.Forward(std::move(input), - std::move(identityLayer.OutputParameter())); + identityLayer.Forward(input, identityLayer.OutputParameter()); module.Add >(identityLayer); } // Test the Forward function. - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(10 * numMergeModules, arma::accu(output)); // Test the Backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(delta)); } } @@ -961,9 +960,9 @@ BOOST_AUTO_TEST_CASE(ReadCellStateParamLSTMLayerTest) input.n_rows, input.n_cols, false, true); // Apply Forward() on LSTM layer. - lstm.Forward(std::move(stepData), // Input. - std::move(outLstm), // Output. - std::move(cellLstm), // Cell state. + lstm.Forward(stepData, // Input. + outLstm, // Output. + cellLstm, // Cell state. false); // Don't write into the cell state. // Compute the value of cell state and output. @@ -1043,9 +1042,9 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) } // Apply Forward() on the LSTM layer. - lstm.Forward(std::move(stepData), // Input. - std::move(outLstm), // Output. - std::move(cellLstm), // Cell state. + lstm.Forward(stepData, // Input. + outLstm, // Output. + cellLstm, // Cell state. true); // Write into cell state. // Compute the value of cell state and output. @@ -1081,18 +1080,18 @@ BOOST_AUTO_TEST_CASE(WriteCellStateParamLSTMLayerTest) arma::mat stepData(input.slice(0).memptr(), input.n_rows, input.n_cols, false, true); - lstm.Forward(std::move(stepData), // Input. - std::move(outLstm), // Output. - std::move(cellLstm), // Cell state. - true); // Write into cell state. + lstm.Forward(stepData, // Input. + outLstm, // Output. + cellLstm, // Cell state. + true); // Write into cell state. for (size_t seqNum = 1; seqNum < rho; ++seqNum) { arma::mat empty; // Should throw error. - BOOST_REQUIRE_THROW(lstm.Forward(std::move(stepData), // Input. - std::move(outLstm), // Output. - std::move(empty), // Cell state. + BOOST_REQUIRE_THROW(lstm.Forward(stepData, // Input. + outLstm, // Output. + empty, // Cell state. true), // Write into cell state. std::runtime_error); } @@ -1149,7 +1148,9 @@ BOOST_AUTO_TEST_CASE(GradientGRULayerTest) */ BOOST_AUTO_TEST_CASE(ForwardGRULayerTest) { - GRU<> gru(3, 3, 5); + // This will make it easier to clean memory later. + GRU<>* gruAlloc = new GRU<>(3, 3, 5); + GRU<>& gru = *gruAlloc; // Initialize the weights to all ones. NetworkInitialization @@ -1160,7 +1161,7 @@ BOOST_AUTO_TEST_CASE(ForwardGRULayerTest) arma::mat input = arma::ones(3, 1); arma::mat output; - gru.Forward(std::move(input), std::move(output)); + gru.Forward(input, output); // Compute the z_t gate output. arma::mat expectedOutput = arma::ones(3, 1); @@ -1175,7 +1176,7 @@ BOOST_AUTO_TEST_CASE(ForwardGRULayerTest) expectedOutput = output; - gru.Forward(std::move(input), std::move(output)); + gru.Forward(input, output); double s = arma::as_scalar(arma::sum(expectedOutput)); @@ -1195,6 +1196,9 @@ BOOST_AUTO_TEST_CASE(ForwardGRULayerTest) expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; BOOST_REQUIRE_LE(arma::as_scalar(arma::trans(output) * expectedOutput), 1e-2); + + LayerTypes<> layer(gruAlloc); + boost::apply_visitor(DeleteVisitor(), layer); } /** @@ -1218,7 +1222,7 @@ BOOST_AUTO_TEST_CASE(SimpleConcatLayerTest) // Test the Forward function. input = arma::zeros(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_CLOSE(arma::accu( moduleA.Parameters().submat(100, 0, moduleA.Parameters().n_elem - 1, 0)) + arma::accu(moduleB.Parameters().submat(100, 0, @@ -1227,7 +1231,7 @@ BOOST_AUTO_TEST_CASE(SimpleConcatLayerTest) // Test the Backward function. error = arma::zeros(20, 1); - module.Backward(std::move(input), std::move(error), std::move(delta)); + module.Backward(input, error, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); } @@ -1260,8 +1264,8 @@ BOOST_AUTO_TEST_CASE(ConcatAlongAxisTest) moduleB.Parameters().randu(); // Compute output of each layer. - moduleA.Forward(std::move(input), std::move(outputA)); - moduleB.Forward(std::move(input), std::move(outputB)); + moduleA.Forward(input, outputA); + moduleB.Forward(input, outputB); arma::cube A(outputA.memptr(), outputWidth, outputHeight, outputChannel); arma::cube B(outputB.memptr(), outputWidth, outputHeight, outputChannel); @@ -1305,7 +1309,7 @@ BOOST_AUTO_TEST_CASE(ConcatAlongAxisTest) Concat<> module(inputSize, axis); module.Add(moduleA); module.Add(moduleB); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); arma::cube concatOut(output.memptr(), x * outputWidth, y * outputHeight, z * outputChannel); @@ -1374,12 +1378,12 @@ BOOST_AUTO_TEST_CASE(SimpleConcatenateLayerTest) module.Concat() = arma::ones(5, 1) * 0.5; // Test the Forward function. - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 7.5); // Test the Backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 5); } @@ -1447,7 +1451,7 @@ BOOST_AUTO_TEST_CASE(SimpleLookupLayerTest) input(0) = 1; input(1) = 3; - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); // The Lookup module uses index - 1 for the cols. const double outputSum = arma::accu(module.Parameters().col(0)) + @@ -1456,7 +1460,7 @@ BOOST_AUTO_TEST_CASE(SimpleLookupLayerTest) BOOST_REQUIRE_CLOSE(outputSum, arma::accu(output), 1e-3); // Test the Backward function. - module.Backward(std::move(input), std::move(input), std::move(delta)); + module.Backward(input, input, delta); BOOST_REQUIRE_EQUAL(arma::accu(input), arma::accu(input)); // Test the Gradient function. @@ -1464,7 +1468,7 @@ BOOST_AUTO_TEST_CASE(SimpleLookupLayerTest) error = error.t(); error.col(1) *= 0.5; - module.Gradient(std::move(input), std::move(error), std::move(gradient)); + module.Gradient(input, error, gradient); // The Lookup module uses index - 1 for the cols. const double gradientSum = arma::accu(gradient.col(0)) + @@ -1484,7 +1488,7 @@ BOOST_AUTO_TEST_CASE(SimpleLogSoftmaxLayerTest) // Test the Forward function. input = arma::mat("0.5; 0.5"); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_SMALL(arma::accu(arma::abs( arma::mat("-0.6931; -0.6931") - output)), 1e-3); @@ -1492,7 +1496,7 @@ BOOST_AUTO_TEST_CASE(SimpleLogSoftmaxLayerTest) error = arma::zeros(input.n_rows, input.n_cols); // Assume LogSoftmax layer is always associated with NLL output layer. error(1, 0) = -1; - module.Backward(std::move(input), std::move(error), std::move(delta)); + module.Backward(input, error, delta); BOOST_REQUIRE_SMALL(arma::accu(arma::abs( arma::mat("1.6487; 0.6487") - delta)), 1e-3); } @@ -1521,13 +1525,12 @@ BOOST_AUTO_TEST_CASE(SimpleBilinearInterpolationLayerTest) 2.0000 2.4000 2.8000 3.0000 3.0000 \ 2.0000 2.4000 2.8000 3.0000 3.0000"); expectedOutput.reshape(25, 1); - layer.Forward(std::move(input), std::move(output)); + layer.Forward(input, output); CheckMatrices(output - expectedOutput, arma::zeros(output.n_rows), 1e-12); expectedOutput = arma::mat("1.0000 1.9000 1.9000 2.8000"); expectedOutput.reshape(4, 1); - layer.Backward(std::move(output), std::move(output), - std::move(unzoomedOutput)); + layer.Backward(output, output, unzoomedOutput); CheckMatrices(unzoomedOutput - expectedOutput, arma::zeros(input.n_rows), 1e-12); } @@ -1549,7 +1552,7 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) // Non-Deteministic Forward Pass Test. model.Deterministic() = false; - model.Forward(std::move(input), std::move(output)); + model.Forward(input, output); arma::mat result; result << 1.1658 << 0.1100 << -1.2758 << arma::endr << 1.2579 << -0.0699 << -1.1880 << arma::endr @@ -1576,7 +1579,7 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) result.clear(); model.Deterministic() = true; - model.Forward(std::move(input), std::move(output)); + model.Forward(input, output); result << 1.1658 << 0.1100 << -1.2757 << arma::endr << 1.2579 << -0.0699 << -1.1880 << arma::endr @@ -1733,12 +1736,12 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) module1.Parameters()(0) = 1.0; module1.Parameters()(8) = 2.0; module1.Reset(); - module1.Forward(std::move(input), std::move(output)); + module1.Forward(input, output); // Value calculated using tensorflow.nn.conv2d_transpose() BOOST_REQUIRE_EQUAL(arma::accu(output), 360.0); // Test the backward function. - module1.Backward(std::move(input), std::move(output), std::move(delta)); + module1.Backward(input, output, delta); // Value calculated using tensorflow.nn.conv2d() BOOST_REQUIRE_EQUAL(arma::accu(delta), 720.0); @@ -1753,12 +1756,12 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) module2.Parameters()(12) = 1.0; module2.Parameters()(15) = 2.0; module2.Reset(); - module2.Forward(std::move(input), std::move(output)); + module2.Forward(input, output); // Value calculated using torch.nn.functional.conv_transpose2d() BOOST_REQUIRE_EQUAL(arma::accu(output), 1512.0); // Test the backward function. - module2.Backward(std::move(input), std::move(output), std::move(delta)); + module2.Backward(input, output, delta); // Value calculated using torch.nn.functional.conv2d() BOOST_REQUIRE_EQUAL(arma::accu(delta), 6504.0); @@ -1771,12 +1774,12 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) module3.Parameters()(3) = 3.0; module3.Parameters()(8) = 1.0; module3.Reset(); - module3.Forward(std::move(input), std::move(output)); + module3.Forward(input, output); // Value calculated using torch.nn.functional.conv_transpose2d() BOOST_REQUIRE_EQUAL(arma::accu(output), 2370.0); // Test the backward function. - module3.Backward(std::move(input), std::move(output), std::move(delta)); + module3.Backward(input, output, delta); // Value calculated using torch.nn.functional.conv2d() BOOST_REQUIRE_EQUAL(arma::accu(delta), 19154.0); @@ -1789,12 +1792,12 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) module4.Parameters()(6) = 6.0; module4.Parameters()(8) = 8.0; module4.Reset(); - module4.Forward(std::move(input), std::move(output)); + module4.Forward(input, output); // Value calculated using torch.nn.functional.conv_transpose2d() BOOST_REQUIRE_EQUAL(arma::accu(output), 6000.0); // Test the backward function. - module4.Backward(std::move(input), std::move(output), std::move(delta)); + module4.Backward(input, output, delta); // Value calculated using torch.nn.functional.conv2d() BOOST_REQUIRE_EQUAL(arma::accu(delta), 86208.0); @@ -1807,12 +1810,12 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) module5.Parameters()(6) = 4.0; module5.Parameters()(8) = 2.0; module5.Reset(); - module5.Forward(std::move(input), std::move(output)); + module5.Forward(input, output); // Value calculated using torch.nn.functional.conv_transpose2d() BOOST_REQUIRE_EQUAL(arma::accu(output), 120.0); // Test the backward function. - module5.Backward(std::move(input), std::move(output), std::move(delta)); + module5.Backward(input, output, delta); // Value calculated using torch.nn.functional.conv2d() BOOST_REQUIRE_EQUAL(arma::accu(delta), 960.0); @@ -1825,12 +1828,12 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) module6.Parameters()(6) = 2.0; module6.Parameters()(8) = 4.0; module6.Reset(); - module6.Forward(std::move(input), std::move(output)); + module6.Forward(input, output); // Value calculated using torch.nn.functional.conv_transpose2d() BOOST_REQUIRE_EQUAL(arma::accu(output), 410.0); // Test the backward function. - module6.Backward(std::move(input), std::move(output), std::move(delta)); + module6.Backward(input, output, delta); // Value calculated using torch.nn.functional.conv2d() BOOST_REQUIRE_EQUAL(arma::accu(delta), 4444.0); @@ -1843,11 +1846,11 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) module7.Parameters()(4) = 2.0; module7.Parameters()(8) = 4.0; module7.Reset(); - module7.Forward(std::move(input), std::move(output)); + module7.Forward(input, output); // Value calculated using torch.nn.functional.conv_transpose2d() BOOST_REQUIRE_EQUAL(arma::accu(output), 606.0); - module7.Backward(std::move(input), std::move(output), std::move(delta)); + module7.Backward(input, output, delta); // Value calculated using torch.nn.functional.conv2d() BOOST_REQUIRE_EQUAL(arma::accu(delta), 7732.0); } @@ -1919,18 +1922,17 @@ BOOST_AUTO_TEST_CASE(SimpleMultiplyMergeLayerTest) for (size_t m = 0; m < numMergeModules; ++m) { IdentityLayer<> identityLayer; - identityLayer.Forward(std::move(input), - std::move(identityLayer.OutputParameter())); + identityLayer.Forward(input, identityLayer.OutputParameter()); module.Add >(identityLayer); } // Test the Forward function. - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_EQUAL(10, arma::accu(output)); // Test the Backward function. - module.Backward(std::move(input), std::move(output), std::move(delta)); + module.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(output), arma::accu(delta)); } } @@ -1949,12 +1951,12 @@ BOOST_AUTO_TEST_CASE(SimpleAtrousConvolutionLayerTest) module1.Parameters()(0) = 1.0; module1.Parameters()(8) = 2.0; module1.Reset(); - module1.Forward(std::move(input), std::move(output)); + module1.Forward(input, output); // Value calculated using tensorflow.nn.atrous_conv2d() BOOST_REQUIRE_EQUAL(arma::accu(output), 792.0); // Test the Backward function. - module1.Backward(std::move(input), std::move(output), std::move(delta)); + module1.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 2376); AtrousConvolution<> module2(1, 1, 3, 3, 2, 2, 0, 0, 7, 7, 2, 2); @@ -1965,12 +1967,12 @@ BOOST_AUTO_TEST_CASE(SimpleAtrousConvolutionLayerTest) module2.Parameters()(3) = 1.0; module2.Parameters()(6) = 1.0; module2.Reset(); - module2.Forward(std::move(input), std::move(output)); + module2.Forward(input, output); // Value calculated using tensorflow.nn.conv2d() BOOST_REQUIRE_EQUAL(arma::accu(output), 264.0); // Test the backward function. - module2.Backward(std::move(input), std::move(output), std::move(delta)); + module2.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 792.0); } @@ -2094,14 +2096,14 @@ BOOST_AUTO_TEST_CASE(AtrousConvolutionLayerPaddingTest) input = arma::linspace(0, 48, 49); module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); module1.Reset(); - module1.Forward(std::move(input), std::move(output)); + module1.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 0); BOOST_REQUIRE_EQUAL(output.n_rows, 9); BOOST_REQUIRE_EQUAL(output.n_cols, 1); // Test the Backward function. - module1.Backward(std::move(input), std::move(output), std::move(delta)); + module1.Backward(input, output, delta); // Check same padding option. AtrousConvolution<> module2(1, 1, 3, 3, 1, 1, @@ -2112,14 +2114,14 @@ BOOST_AUTO_TEST_CASE(AtrousConvolutionLayerPaddingTest) input = arma::linspace(0, 48, 49); module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); module2.Reset(); - module2.Forward(std::move(input), std::move(output)); + module2.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 0); BOOST_REQUIRE_EQUAL(output.n_rows, 49); BOOST_REQUIRE_EQUAL(output.n_cols, 1); // Test the backward function. - module2.Backward(std::move(input), std::move(output), std::move(delta)); + module2.Backward(input, output, delta); } /** @@ -2135,7 +2137,7 @@ BOOST_AUTO_TEST_CASE(LayerNormTest) LayerNorm<> model(input.n_rows); model.Reset(); - model.Forward(std::move(input), std::move(output)); + model.Forward(input, output); arma::mat result; result << 1.2247 << 1.2978 << arma::endr << 0 << -1.1355 << arma::endr @@ -2218,13 +2220,13 @@ BOOST_AUTO_TEST_CASE(AddMergeRunTest) linear->Reset(); input = arma::zeros(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); double parameterSum = arma::accu(linear->Parameters().submat( 100, 0, linear->Parameters().n_elem - 1, 0)); // Test the Backward function. - module.Backward(std::move(input), std::move(input), std::move(delta)); + module.Backward(input, input, delta); // Clean up before we break, delete linear; @@ -2250,13 +2252,13 @@ BOOST_AUTO_TEST_CASE(MultiplyMergeRunTest) linear->Reset(); input = arma::zeros(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); double parameterSum = arma::accu(linear->Parameters().submat( 100, 0, linear->Parameters().n_elem - 1, 0)); // Test the Backward function. - module.Backward(std::move(input), std::move(input), std::move(delta)); + module.Backward(input, input, delta); // Clean up before we break, delete linear; @@ -2275,19 +2277,19 @@ BOOST_AUTO_TEST_CASE(SimpleSubviewLayerTest) // Test the Forward function for a vector. input = arma::ones(20, 1); - moduleRow.Forward(std::move(input), std::move(output)); + moduleRow.Forward(input, output); BOOST_REQUIRE_EQUAL(output.n_rows, 10); Subview<> moduleMat(4, 3, 6, 0, 2); // Test the Forward function for a matrix. input = arma::ones(20, 8); - moduleMat.Forward(std::move(input), std::move(outputMat)); + moduleMat.Forward(input, outputMat); BOOST_REQUIRE_EQUAL(outputMat.n_rows, 12); BOOST_REQUIRE_EQUAL(outputMat.n_cols, 2); // Test the Backward function. - moduleMat.Backward(std::move(input), std::move(input), std::move(delta)); + moduleMat.Backward(input, input, delta); BOOST_REQUIRE_EQUAL(accu(delta), 160); BOOST_REQUIRE_EQUAL(delta.n_rows, 20); } @@ -2304,21 +2306,21 @@ BOOST_AUTO_TEST_CASE(SubviewIndexTest) Subview<> moduleStart(1, 0, 9); arma::mat subStart = arma::linspace(1, 10, 10); - moduleStart.Forward(std::move(input), std::move(outputStart)); + moduleStart.Forward(input, outputStart); CheckMatrices(outputStart, subStart); // Slicing from the mid indices. Subview<> moduleMid(1, 6, 15); arma::mat subMid = arma::linspace(7, 16, 10); - moduleMid.Forward(std::move(input), std::move(outputMid)); + moduleMid.Forward(input, outputMid); CheckMatrices(outputMid, subMid); // Slicing from the end indices. Subview<> moduleEnd(1, 10, 19); arma::mat subEnd = arma::linspace(11, 20, 10); - moduleEnd.Forward(std::move(input), std::move(outputEnd)); + moduleEnd.Forward(input, outputEnd); CheckMatrices(outputEnd, subEnd); } @@ -2334,14 +2336,14 @@ BOOST_AUTO_TEST_CASE(SubviewBatchTest) // Test with inSize 1. input = arma::ones(20, 8); - moduleCol.Forward(std::move(input), std::move(outputCol)); + moduleCol.Forward(input, outputCol); CheckMatrices(outputCol, input); // Few rows and columns selected. Subview<> moduleMat(4, 3, 6, 0, 2); // Test with inSize greater than 1. - moduleMat.Forward(std::move(input), std::move(outputMat)); + moduleMat.Forward(input, outputMat); output = arma::ones(12, 2); CheckMatrices(outputMat, output); @@ -2349,7 +2351,7 @@ BOOST_AUTO_TEST_CASE(SubviewBatchTest) Subview<> moduleDef(4, 1, 6, 0, 4); // Test with inSize greater than 1 and endCol >= inSize. - moduleDef.Forward(std::move(input), std::move(outputDef)); + moduleDef.Forward(input, outputDef); output = arma::ones(24, 2); CheckMatrices(outputDef, output); } @@ -2367,12 +2369,12 @@ BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerTest) // output should be small enough. input = join_cols(arma::ones(5, 1) * -15, arma::zeros(5, 1)); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); BOOST_REQUIRE_LE(arma::accu(output), 1e-5); // Test the Backward function. arma::mat gy = arma::zeros(5, 1); - module.Backward(std::move(input), std::move(gy), std::move(delta)); + module.Backward(input, gy, delta); BOOST_REQUIRE(arma::accu(delta) != 0); // klBackward will be added. } @@ -2388,8 +2390,8 @@ BOOST_AUTO_TEST_CASE(ReparametrizationLayerStochasticTest) arma::zeros(5, 1)); // Test if two forward passes generate same output. - module.Forward(std::move(input), std::move(outputA)); - module.Forward(std::move(input), std::move(outputB)); + module.Forward(input, outputA); + module.Forward(input, outputB); CheckMatrices(outputA, outputB); } @@ -2404,14 +2406,14 @@ BOOST_AUTO_TEST_CASE(ReparametrizationLayerIncludeKlTest) input = join_cols(arma::ones(5, 1), arma::zeros(5, 1)); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); // As KL divergence is not included, with the above inputs, the delta // matrix should be all zeros. gy = arma::zeros(output.n_rows, output.n_cols); - module.Backward(std::move(output), std::move(gy), std::move(delta)); + module.Backward(output, gy, delta); - BOOST_REQUIRE_EQUAL(arma::accu(std::move(delta)), 0); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); } /** @@ -2549,14 +2551,14 @@ BOOST_AUTO_TEST_CASE(SimpleResidualLayerTest) // Test the Forward function (pass the same input to both). input = arma::randu(10, 1); - sequential->Forward(std::move(input), std::move(outputA)); - residual->Forward(std::move(input), std::move(outputB)); + sequential->Forward(input, outputA); + residual->Forward(input, outputB); CheckMatrices(outputA, outputB - input); // Test the Backward function (pass the same error to both). - sequential->Backward(std::move(input), std::move(input), std::move(deltaA)); - residual->Backward(std::move(input), std::move(input), std::move(deltaB)); + sequential->Backward(input, input, deltaA); + residual->Backward(input, input, deltaB); CheckMatrices(deltaA, deltaB - input); @@ -2593,8 +2595,8 @@ BOOST_AUTO_TEST_CASE(SimpleHighwayLayerTest) // Test the Forward function (pass the same input to both). input = arma::randu(10, 1); - sequential->Forward(std::move(input), std::move(outputA)); - highway->Forward(std::move(input), std::move(outputB)); + sequential->Forward(input, outputA); + highway->Forward(input, outputB); CheckMatrices(outputB, input * 0.5 + outputA * 0.5); @@ -2636,7 +2638,6 @@ BOOST_AUTO_TEST_CASE(GradientHighwayLayerTest) ~GradientFunction() { - highway->DeleteModules(); delete model; } @@ -2688,7 +2689,6 @@ BOOST_AUTO_TEST_CASE(GradientSequentialLayerTest) ~GradientFunction() { - sequential->DeleteModules(); delete model; } @@ -2774,10 +2774,10 @@ BOOST_AUTO_TEST_CASE(WeightNormRunTest) linear->Bias().zeros(); input = arma::zeros(10, 1); - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); // Test the Backward function. - module.Backward(std::move(input), std::move(input), std::move(delta)); + module.Backward(input, input, delta); BOOST_REQUIRE_EQUAL(0, arma::accu(output)); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); @@ -2904,14 +2904,14 @@ BOOST_AUTO_TEST_CASE(ConvolutionLayerPaddingTest) input = arma::linspace(0, 48, 49); module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); module1.Reset(); - module1.Forward(std::move(input), std::move(output)); + module1.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 0); BOOST_REQUIRE_EQUAL(output.n_rows, 25); BOOST_REQUIRE_EQUAL(output.n_cols, 1); // Test the Backward function. - module1.Backward(std::move(input), std::move(output), std::move(delta)); + module1.Backward(input, output, delta); // Check same padding option. Convolution<> module2(1, 1, 3, 3, 1, 1, std::tuple(0, 0), @@ -2921,14 +2921,14 @@ BOOST_AUTO_TEST_CASE(ConvolutionLayerPaddingTest) input = arma::linspace(0, 48, 49); module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); module2.Reset(); - module2.Forward(std::move(input), std::move(output)); + module2.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 0); BOOST_REQUIRE_EQUAL(output.n_rows, 49); BOOST_REQUIRE_EQUAL(output.n_cols, 1); // Test the backward function. - module2.Backward(std::move(input), std::move(output), std::move(delta)); + module2.Backward(input, output, delta); } /** @@ -2944,12 +2944,12 @@ BOOST_AUTO_TEST_CASE(TransposedConvolutionLayerPaddingTest) input = arma::linspace(0, 15, 16); module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); module1.Reset(); - module1.Forward(std::move(input), std::move(output)); + module1.Forward(input, output); // Value calculated using tensorflow.nn.conv2d_transpose(). BOOST_REQUIRE_EQUAL(arma::accu(output), 0.0); // Test the Backward Function. - module1.Backward(std::move(input), std::move(output), std::move(delta)); + module1.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); // Test Valid for non zero padding. @@ -2964,12 +2964,12 @@ BOOST_AUTO_TEST_CASE(TransposedConvolutionLayerPaddingTest) module2.Parameters()(6) = 4.0; module2.Parameters()(8) = 2.0; module2.Reset(); - module2.Forward(std::move(input), std::move(output)); + module2.Forward(input, output); // Value calculated using torch.nn.functional.conv_transpose2d(). BOOST_REQUIRE_EQUAL(arma::accu(output), 120.0); // Test the Backward Function. - module2.Backward(std::move(input), std::move(output), std::move(delta)); + module2.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 960.0); // Test for same padding type. @@ -2978,13 +2978,13 @@ BOOST_AUTO_TEST_CASE(TransposedConvolutionLayerPaddingTest) input = arma::linspace(0, 8, 9); module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); module3.Reset(); - module3.Forward(std::move(input), std::move(output)); + module3.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 0); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); // Test the Backward Function. - module3.Backward(std::move(input), std::move(output), std::move(delta)); + module3.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); // Output shape should equal input. @@ -2995,13 +2995,13 @@ BOOST_AUTO_TEST_CASE(TransposedConvolutionLayerPaddingTest) input = arma::linspace(0, 24, 25); module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); module4.Reset(); - module4.Forward(std::move(input), std::move(output)); + module4.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 0); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); // Test the Backward Function. - module4.Backward(std::move(input), std::move(output), std::move(delta)); + module4.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); TransposedConvolution<> module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 2, 2, "SAME"); @@ -3009,13 +3009,13 @@ BOOST_AUTO_TEST_CASE(TransposedConvolutionLayerPaddingTest) input = arma::linspace(0, 3, 4); module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); module5.Reset(); - module5.Forward(std::move(input), std::move(output)); + module5.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 0); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); // Test the Backward Function. - module5.Backward(std::move(input), std::move(output), std::move(delta)); + module5.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); TransposedConvolution<> module6(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 5, 5, "SAME"); @@ -3023,13 +3023,98 @@ BOOST_AUTO_TEST_CASE(TransposedConvolutionLayerPaddingTest) input = arma::linspace(0, 24, 25); module6.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); module6.Reset(); - module6.Forward(std::move(input), std::move(output)); + module6.Forward(input, output); BOOST_REQUIRE_EQUAL(arma::accu(output), 0); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); // Test the Backward Function. - module6.Backward(std::move(input), std::move(output), std::move(delta)); + module6.Backward(input, output, delta); BOOST_REQUIRE_EQUAL(arma::accu(delta), 0.0); } + +/** + * Simple test for Max Pooling layer. + */ +BOOST_AUTO_TEST_CASE(MaxPoolingTestCase) +{ + // For rectangular input to pooling layers. + arma::mat input = arma::mat(12, 1); + arma::mat output; + input.zeros(); + input(0) = 1; + input(1) = 2; + input(2) = 3; + input(3) = input(8) = 7; + input(4) = 4; + input(5) = 5; + input(6) = input(7) = 6; + input(10) = 8; + input(11) = 9; + // Output-Size should be 2 x 2. + // Square output. + MaxPooling<> module1(2, 2, 2, 1); + module1.InputHeight() = 3; + module1.InputWidth() = 4; + module1.Forward(input, output); + // Calculated using torch.nn.MaxPool2d(). + BOOST_REQUIRE_EQUAL(arma::accu(output), 28); + BOOST_REQUIRE_EQUAL(output.n_elem, 4); + BOOST_REQUIRE_EQUAL(output.n_cols, 1); + + // For Square input. + input = arma::mat(9, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(3) = 3; + input(6) = 3; + // Output-Size should be 1 x 2. + // Rectangular output. + MaxPooling<> module2(3, 2, 3, 1); + module2.InputHeight() = 3; + module2.InputWidth() = 3; + module2.Forward(input, output); + // Calculated using torch.nn.MaxPool2d(). + BOOST_REQUIRE_EQUAL(arma::accu(output), 12.0); + BOOST_REQUIRE_EQUAL(output.n_elem, 2); + BOOST_REQUIRE_EQUAL(output.n_cols, 1); + + // For Square input. + input = arma::mat(16, 1); + input.zeros(); + input(0) = 6; + input(1) = 3; + input(2) = 9; + input(4) = 3; + input(8) = 3; + // Output-Size should be 3 x 3. + // Square output. + MaxPooling<> module3(2, 2, 1, 1); + module3.InputHeight() = 4; + module3.InputWidth() = 4; + module3.Forward(input, output); + // Calculated using torch.nn.MaxPool2d(). + BOOST_REQUIRE_EQUAL(arma::accu(output), 30.0); + BOOST_REQUIRE_EQUAL(output.n_elem, 9); + BOOST_REQUIRE_EQUAL(output.n_cols, 1); + + // For Rectangular input. + input = arma::mat(6, 1); + input.zeros(); + input(0) = 1; + input(1) = 1; + input(3) = 1; + // Output-Size should be 2 x 2. + // Square output. + MaxPooling<> module4(2, 1, 1, 1); + module4.InputHeight() = 2; + module4.InputWidth() = 3; + module4.Forward(input, output); + // Calculated using torch.nn.MaxPool2d(). + BOOST_REQUIRE_EQUAL(arma::accu(output), 3); + BOOST_REQUIRE_EQUAL(output.n_elem, 4); + BOOST_REQUIRE_EQUAL(output.n_cols, 1); +} BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp index be789befd5..00a4c80810 100644 --- a/src/mlpack/tests/ann_test_tools.hpp +++ b/src/mlpack/tests/ann_test_tools.hpp @@ -53,7 +53,7 @@ double JacobianTest(ModuleType& module, ResetFunction(module); // Initialize the jacobian matrix. - module.Forward(std::move(input), std::move(output)); + module.Forward(input, output); jacobianA = arma::zeros(input.n_elem, output.n_elem); // Share the input paramter matrix. @@ -64,9 +64,9 @@ double JacobianTest(ModuleType& module, { double original = sin(i); sin(i) = original - perturbation; - module.Forward(std::move(input), std::move(outputA)); + module.Forward(input, outputA); sin(i) = original + perturbation; - module.Forward(std::move(input), std::move(outputB)); + module.Forward(input, outputB); sin(i) = original; outputB -= outputA; @@ -90,7 +90,7 @@ double JacobianTest(ModuleType& module, derivTemp(i) = 1; arma::mat delta; - module.Backward(std::move(input), std::move(deriv), std::move(delta)); + module.Backward(input, deriv, delta); jacobianB.col(i) = delta; } @@ -106,10 +106,10 @@ double JacobianPerformanceTest(ModuleType& module, arma::mat& target, const double eps = 1e-6) { - module.Forward(std::move(input), std::move(target)); + module.Forward(input, target); arma::mat delta; - module.Backward(std::move(input), std::move(target), std::move(delta)); + module.Backward(input, target, delta); arma::mat centralDifference = arma::zeros(delta.n_rows, delta.n_cols); arma::mat inputTemp = arma::mat(input.memptr(), input.n_rows, input.n_cols, @@ -121,9 +121,9 @@ double JacobianPerformanceTest(ModuleType& module, for (size_t i = 0; i < input.n_elem; ++i) { inputTemp(i) = inputTemp(i) + eps; - double outputA = module.Forward(std::move(input), std::move(target)); + double outputA = module.Forward(input, target); inputTemp(i) = inputTemp(i) - (2 * eps); - double outputB = module.Forward(std::move(input), std::move(target)); + double outputB = module.Forward(input, target); centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps); inputTemp(i) = inputTemp(i) + eps; diff --git a/src/mlpack/tests/ann_visitor_test.cpp b/src/mlpack/tests/ann_visitor_test.cpp index e7db9708b2..e79dc4b5b2 100644 --- a/src/mlpack/tests/ann_visitor_test.cpp +++ b/src/mlpack/tests/ann_visitor_test.cpp @@ -35,24 +35,24 @@ BOOST_AUTO_TEST_CASE(BiasSetVisitorTest) ResetVisitor resetVisitor; - boost::apply_visitor(WeightSetVisitor(std::move(layerWeights), 0), linear); + boost::apply_visitor(WeightSetVisitor(layerWeights, 0), linear); boost::apply_visitor(resetVisitor, linear); arma::mat weight = {"1 2 3 4 5 6 7 8 9 10"}; - size_t biasSize = boost::apply_visitor(BiasSetVisitor(std::move(weight), - 0), linear); + size_t biasSize = boost::apply_visitor(BiasSetVisitor(weight, 0), linear); BOOST_REQUIRE_EQUAL(biasSize, 10); arma::mat input(10, 1), output; input.randu(); - boost::apply_visitor(ForwardVisitor(std::move(input), std::move(output)), - linear); + boost::apply_visitor(ForwardVisitor(input, output), linear); BOOST_REQUIRE_EQUAL(arma::accu(output), 55); + + boost::apply_visitor(DeleteVisitor(), linear); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/callback_test.cpp b/src/mlpack/tests/callback_test.cpp index 6bd3db2147..a75b647b40 100644 --- a/src/mlpack/tests/callback_test.cpp +++ b/src/mlpack/tests/callback_test.cpp @@ -253,6 +253,7 @@ BOOST_AUTO_TEST_CASE(RBMCallbackTest) // Call the train function with printloss callback. double objVal = model.Train(msgd, ens::ProgressBar(70, stream)); + BOOST_REQUIRE(!std::isnan(objVal)); BOOST_REQUIRE_GT(stream.str().length(), 0); } diff --git a/src/mlpack/tests/convolutional_network_test.cpp b/src/mlpack/tests/convolutional_network_test.cpp index 6c0f0059f1..a7c3e41442 100644 --- a/src/mlpack/tests/convolutional_network_test.cpp +++ b/src/mlpack/tests/convolutional_network_test.cpp @@ -114,13 +114,7 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; } - size_t correct = 0; - for (size_t i = 0; i < X.n_cols; i++) - { - if (prediction(i) == Y(i)) - correct++; - } - + size_t correct = arma::accu(prediction == Y); double classificationError = 1 - double(correct) / X.n_cols; if (classificationError <= 0.25) { diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index b4182ac1fb..4210065887 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -164,6 +165,28 @@ BOOST_AUTO_TEST_CASE(MSETest) BOOST_REQUIRE_CLOSE(MSE::Evaluate(lr, data, responses), expectedMSE, 1e-5); } +/** + * Test the R squared metric (R2 Score). + */ +BOOST_AUTO_TEST_CASE(R2ScoreTest) +{ + // Making two points that define the linear function f(x) = x - 1. + arma::mat trainingData("0 1"); + arma::rowvec trainingResponses("-1 0"); + + LinearRegression lr(trainingData, trainingResponses); + + // Making five responses that are the output of regression function f(x) + // with some responses having a slight deviation of 0.005. + // Mean Responses = (1 + 2 + 3 + 6 + 8)/5 = 4. + arma::mat data("2 3 4 7 9"); + arma::rowvec responses("1 2.005 3 6.005 8.005"); + + double expectedR2 = 0.99999779; + + BOOST_REQUIRE_CLOSE(R2Score::Evaluate(lr, data, responses), expectedR2, 1e-5); +} + /** * Test the mean squared error with matrix responses. */ diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index b84f2a557b..c7c8ef6fdb 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -53,17 +53,8 @@ void TestNetwork(ModelType& model, arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; } - size_t error = 0; - for (size_t i = 0; i < testData.n_cols; i++) - { - if (int(arma::as_scalar(prediction.col(i))) == - int(arma::as_scalar(testLabels.col(i)))) - { - error++; - } - } - - double classificationError = 1 - double(error) / testData.n_cols; + size_t correct = arma::accu(prediction == testLabels); + double classificationError = 1 - double(correct) / testData.n_cols; BOOST_REQUIRE_LE(classificationError, classificationErrorThreshold); } @@ -183,7 +174,7 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest) arma::mat currentResuls; model.Forward(currentData, currentResuls); arma::mat gradients; - model.Backward(currentLabels, gradients); + model.Backward(currentData, currentLabels, gradients); #if ENS_VERSION_MAJOR == 1 opt.Update(model.Parameters(), stepSize, gradients); #else @@ -199,16 +190,7 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest) arma::max(currentResuls.col(i)) == currentResuls.col(i), 1)) + 1; } - size_t correct = 0; - for (size_t i = 0; i < currentLabels.n_cols; i++) - { - if (int(arma::as_scalar(prediction.col(i))) == - int(arma::as_scalar(currentLabels.col(i)))) - { - correct++; - } - } - + size_t correct = arma::accu(prediction == currentLabels); error(1 - (double) correct / batchSize); } Log::Debug << "Current training error: " << error.mean() << std::endl; @@ -318,7 +300,7 @@ BOOST_AUTO_TEST_CASE(HighwayNetworkTest) Highway<>* highway = new Highway<>(10, true); highway->Add >(10, 10); highway->Add >(); - model.Add(highway); + model.Add(highway); // This takes ownership of the memory. model.Add >(10, 2); model.Add >(); TestNetwork<>(model, dataset, labels, dataset, labels, 10, 0.2); @@ -600,10 +582,8 @@ BOOST_AUTO_TEST_CASE(FFNReturnModel) // Get the layer parameter from layer A and layer B and store them in // parameterA and parameterB. arma::mat parameterA, parameterB; - boost::apply_visitor(ParametersVisitor(std::move(parameterA)), - model.Model()[0]); - boost::apply_visitor(ParametersVisitor(std::move(parameterB)), - model.Model()[1]); + boost::apply_visitor(ParametersVisitor(parameterA), model.Model()[0]); + boost::apply_visitor(ParametersVisitor(parameterB), model.Model()[1]); CheckMatrices(parameterA, arma::ones(3 * 3 + 3, 1)); CheckMatrices(parameterB, arma::zeros(3 * 4 + 4, 1)); @@ -612,4 +592,33 @@ BOOST_AUTO_TEST_CASE(FFNReturnModel) CheckMatrices(linearB->Parameters(), arma::zeros(3 * 4 + 4, 1)); } +/** + * Test to see if the FFN code compiles when the Optimizer + * doesn't have the MaxIterations() method. + */ +BOOST_AUTO_TEST_CASE(OptimizerTest) +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + + FFN, RandomInitialization, CustomLayer<> > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(8, 3); + model.Add >(); + + ens::DE opt(200, 1000, 0.6, 0.8, 1e-5); + model.Train(trainData, trainLabels, opt); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index 0e2bbb0459..b5172dbeb0 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -12,6 +12,8 @@ #include #include +#include "test_tools.hpp" +#include "serialization.hpp" using namespace mlpack; using namespace mlpack::data; @@ -29,8 +31,10 @@ BOOST_AUTO_TEST_CASE(LoadInvalidExtensionFile) { arma::Mat matrix; data::ImageInfo info; + Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(data::Load("invalidExtendion.p4ng", matrix, info, - false, true), std::runtime_error); + true), std::runtime_error); + Log::Fatal.ignoreInput = false; } /** @@ -40,9 +44,12 @@ BOOST_AUTO_TEST_CASE(LoadImageAPITest) { arma::Mat matrix; data::ImageInfo info; - BOOST_REQUIRE(data::Load("test_image.png", matrix, info, false, - true) == true); - BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); // width * height * channels. + BOOST_REQUIRE(data::Load("test_image.png", matrix, info, false) == true); + // width * height * channels. + BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); + BOOST_REQUIRE_EQUAL(info.Height(), 50); + BOOST_REQUIRE_EQUAL(info.Width(), 50); + BOOST_REQUIRE_EQUAL(info.Channels(), 3); BOOST_REQUIRE_EQUAL(matrix.n_cols, 1); } @@ -56,15 +63,98 @@ BOOST_AUTO_TEST_CASE(SaveImageAPITest) arma::Mat im1; size_t dimension = info.Width() * info.Height() * info.Channels(); im1 = arma::randi>(dimension, 1); - BOOST_REQUIRE(data::Save("APITest.bmp", im1, info, false, true) == true); + BOOST_REQUIRE(data::Save("APITest.bmp", im1, info, false) == true); arma::Mat im2; - BOOST_REQUIRE(data::Load("APITest.bmp", im2, info, false, true) == true); + BOOST_REQUIRE(data::Load("APITest.bmp", im2, info, false) == true); BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); - for (size_t i = 10; i < im1.n_elem; ++i) + for (size_t i = 0; i < im1.n_elem; ++i) BOOST_REQUIRE_EQUAL(im1[i], im2[i]); + remove("APITest.bmp"); +} + +/** + * Test if an image with a wrong dimesion throws an expected + * exception while saving. + */ +BOOST_AUTO_TEST_CASE(SaveImageWrongInfo) +{ + data::ImageInfo info(5, 5, 3, 90); + + arma::Mat im1; + size_t dimension = info.Width() * info.Height() * info.Channels(); + im1 = arma::randi>(24 * 25 * 7, 1); + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(data::Save("APITest.bmp", im1, info, false), + std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * Test that the image is loaded correctly into the matrix using the API + * for vectors. + */ +BOOST_AUTO_TEST_CASE(LoadVectorImageAPITest) +{ + arma::Mat matrix; + data::ImageInfo info; + std::vector files = {"test_image.png", "test_image.png"}; + BOOST_REQUIRE(data::Load(files, matrix, info, false) == true); + // width * height * channels. + BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); + BOOST_REQUIRE_EQUAL(info.Height(), 50); + BOOST_REQUIRE_EQUAL(info.Width(), 50); + BOOST_REQUIRE_EQUAL(info.Channels(), 3); + BOOST_REQUIRE_EQUAL(matrix.n_cols, 2); +} + +/** + * Test if the image is saved correctly using API for arma mat. + */ +BOOST_AUTO_TEST_CASE(SaveImageMatAPITest) +{ + data::ImageInfo info(5, 5, 3); + + arma::Mat im1; + size_t dimension = info.Width() * info.Height() * info.Channels(); + im1 = arma::randi>(dimension, 1); + arma::mat input = arma::conv_to::from(im1); + BOOST_REQUIRE(Save("APITest.bmp", input, info, false) == true); + + arma::mat output; + BOOST_REQUIRE(Load("APITest.bmp", output, info, false) == true); + + BOOST_REQUIRE_EQUAL(input.n_cols, output.n_cols); + BOOST_REQUIRE_EQUAL(input.n_rows, output.n_rows); + for (size_t i = 0; i < input.n_elem; ++i) + BOOST_REQUIRE_CLOSE(input[i], output[i], 1e-5); + remove("APITest.bmp"); +} + +/** + * Serialization test for the ImageInfo class. + */ +BOOST_AUTO_TEST_CASE(ImageInfoSerialization) +{ + data::ImageInfo info(5, 5, 3, 90); + data::ImageInfo xmlInfo, textInfo, binaryInfo; + + SerializeObjectAll(info, xmlInfo, textInfo, binaryInfo); + + BOOST_REQUIRE_EQUAL(info.Width(), xmlInfo.Width()); + BOOST_REQUIRE_EQUAL(info.Height(), xmlInfo.Height()); + BOOST_REQUIRE_EQUAL(info.Channels(), xmlInfo.Channels()); + BOOST_REQUIRE_EQUAL(info.Quality(), xmlInfo.Quality()); + BOOST_REQUIRE_EQUAL(info.Width(), textInfo.Width()); + BOOST_REQUIRE_EQUAL(info.Height(), textInfo.Height()); + BOOST_REQUIRE_EQUAL(info.Channels(), textInfo.Channels()); + BOOST_REQUIRE_EQUAL(info.Quality(), textInfo.Quality()); + BOOST_REQUIRE_EQUAL(info.Width(), binaryInfo.Width()); + BOOST_REQUIRE_EQUAL(info.Height(), binaryInfo.Height()); + BOOST_REQUIRE_EQUAL(info.Channels(), binaryInfo.Channels()); + BOOST_REQUIRE_EQUAL(info.Quality(), binaryInfo.Quality()); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index fd908a8ad6..6b7edc267e 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -353,7 +353,7 @@ BOOST_AUTO_TEST_CASE(TrainingConstructorWithNonDefaultsTest) } /** - * Test that LARS::Train() returns finite correlation value. + * Test that LARS::Train() returns finite error value. */ BOOST_AUTO_TEST_CASE(LARSTrainReturnCorrelation) { @@ -371,30 +371,54 @@ BOOST_AUTO_TEST_CASE(LARSTrainReturnCorrelation) // Test with Cholesky decomposition and with lasso. LARS lars1(true, lambda1, 0.0); arma::vec betaOpt1; - double maxCorr = lars1.Train(X, y, betaOpt1); + double error = lars1.Train(X, y, betaOpt1); - BOOST_REQUIRE_EQUAL(std::isfinite(maxCorr), true); + BOOST_REQUIRE_EQUAL(std::isfinite(error), true); // Test without Cholesky decomposition and with lasso. LARS lars2(false, lambda1, 0.0); arma::vec betaOpt2; - maxCorr = lars2.Train(X, y, betaOpt2); + error = lars2.Train(X, y, betaOpt2); - BOOST_REQUIRE_EQUAL(std::isfinite(maxCorr), true); + BOOST_REQUIRE_EQUAL(std::isfinite(error), true); // Test with Cholesky decomposition and with elasticnet. LARS lars3(true, lambda1, lambda2); arma::vec betaOpt3; - maxCorr = lars3.Train(X, y, betaOpt3); + error = lars3.Train(X, y, betaOpt3); - BOOST_REQUIRE_EQUAL(std::isfinite(maxCorr), true); + BOOST_REQUIRE_EQUAL(std::isfinite(error), true); // Test without Cholesky decomposition and with elasticnet. LARS lars4(false, lambda1, lambda2); arma::vec betaOpt4; - maxCorr = lars4.Train(X, y, betaOpt4); + error = lars4.Train(X, y, betaOpt4); - BOOST_REQUIRE_EQUAL(std::isfinite(maxCorr), true); + BOOST_REQUIRE_EQUAL(std::isfinite(error), true); +} + +/** + * Test that LARS::ComputeError() returns error value less than 1 + * and greater than 0. + */ +BOOST_AUTO_TEST_CASE(LARSTestComputeError) +{ + arma::mat X; + arma::mat Y; + + data::Load("lars_dependent_x.csv", X); + data::Load("lars_dependent_y.csv", Y); + + arma::rowvec y = Y.row(0); + + LARS lars1(true, 0.1, 0.0); + arma::vec betaOpt1; + double train1 = lars1.Train(X, y, betaOpt1); + double cost = lars1.ComputeError(X, y); + + BOOST_REQUIRE_EQUAL(cost <= 1, true); + BOOST_REQUIRE_EQUAL(cost >= 0, true); + BOOST_REQUIRE_EQUAL(cost == train1, true); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index cdcb466170..145ef3b418 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -15,16 +15,20 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include #include #include +#include +#include #include #include @@ -37,6 +41,33 @@ using namespace mlpack::ann; BOOST_AUTO_TEST_SUITE(LossFunctionsTest); +/** + * Simple Huber Loss test. + */ +BOOST_AUTO_TEST_CASE(HuberLossTest) +{ + arma::mat input, target, output; + HuberLoss<> module; + + // Test the Forward function. + input = arma::mat("17.45 12.91 13.63 29.01 7.12 15.47 31.52 31.97"); + target = arma::mat("16.52 13.11 13.67 29.51 24.31 15.03 30.72 34.07"); + double loss = module.Forward(input, target); + BOOST_REQUIRE_CLOSE_FRACTION(loss, 2.410631, 0.00001); + + // Test the backward function. + module.Backward(input, target, output); + + // Expected Output: + // [0.1162 -0.0250 -0.0050 -0.0625 -0.1250 0.0550 0.1000 -0.1250] + // Sum of Expected Output = -0.07125. + double expectedOutputSum = arma::accu(output); + BOOST_REQUIRE_CLOSE_FRACTION(expectedOutputSum, -0.07125, 0.00001); + + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); +} + /** * Simple KL Divergence test. The loss should be zero if input = target. */ @@ -49,7 +80,7 @@ BOOST_AUTO_TEST_CASE(SimpleKLDivergenceTest) // Test the Forward function. Loss should be 0 if input = target. input = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input, target); BOOST_REQUIRE_SMALL(loss, 0.00001); } @@ -65,11 +96,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredLogarithmicErrorTest) // the manually calculated result. input = arma::zeros(1, 8); target = arma::zeros(1, 8); - double error = module.Forward(std::move(input), std::move(target)); + double error = module.Forward(input, target); BOOST_REQUIRE_SMALL(error, 0.00001); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); // The output should be equal to 0. CheckMatrices(input, output); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); @@ -78,11 +109,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredLogarithmicErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(std::move(input), std::move(target)); + error = module.Forward(input, target); BOOST_REQUIRE_CLOSE(error, 0.082760974810151655, 0.001); // Test the Backward function on a single input. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); BOOST_REQUIRE_CLOSE(arma::accu(output), -0.1917880483011872, 0.001); BOOST_REQUIRE_EQUAL(output.n_elem, 1); } @@ -100,11 +131,11 @@ BOOST_AUTO_TEST_CASE(KLDivergenceMeanTest) input = arma::mat("1 1 1 1 1 1 1 1 1 1"); target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input, target); BOOST_REQUIRE_CLOSE_FRACTION(loss, -1.1 , 0.00001); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); BOOST_REQUIRE_CLOSE_FRACTION(arma::as_scalar(output), -0.1, 0.00001); } @@ -121,11 +152,11 @@ BOOST_AUTO_TEST_CASE(KLDivergenceNoMeanTest) input = arma::mat("1 1 1 1 1 1 1 1 1 1"); target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input, target); BOOST_REQUIRE_CLOSE_FRACTION(loss, -11, 0.00001); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); BOOST_REQUIRE_CLOSE_FRACTION(arma::as_scalar(output), -1, 0.00001); } @@ -141,11 +172,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest) // the manually calculated result. input = arma::mat("1.0 0.0 1.0 0.0 -1.0 0.0 -1.0 0.0"); target = arma::zeros(1, 8); - double error = module.Forward(std::move(input), std::move(target)); + double error = module.Forward(input, target); BOOST_REQUIRE_EQUAL(error, 0.5); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); // We subtract a zero vector, so according to the used backward formula: // output = 2 * (input - target) / target.n_cols, // output * nofColumns / 2 should be equal to input. @@ -156,11 +187,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(std::move(input), std::move(target)); + error = module.Forward(input, target); BOOST_REQUIRE_EQUAL(error, 1.0); // Test the Backward function on a single input. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); // Test whether the output is negative. BOOST_REQUIRE_EQUAL(arma::accu(output), -2); BOOST_REQUIRE_EQUAL(output.n_elem, 1); @@ -178,16 +209,16 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest) // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(std::move(input1), std::move(target1)); + double error1 = module.Forward(input1, target1); BOOST_REQUIRE_SMALL(error1 - 8 * std::log(2), 2e-5); input2 = arma::mat("0 1 1 0 1 0 0 1"); target2 = arma::mat("0 1 1 0 1 0 0 1"); - double error2 = module.Forward(std::move(input2), std::move(target2)); + double error2 = module.Forward(input2, target2); BOOST_REQUIRE_SMALL(error2, 1e-5); // Test the Backward function. - module.Backward(std::move(input1), std::move(target1), std::move(output)); + module.Backward(input1, target1, output); for (double el : output) { // For the 0.5 constant vector we should get 1 / (1 - 0.5) = 2 everywhere. @@ -196,7 +227,7 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest) BOOST_REQUIRE_EQUAL(output.n_rows, input1.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); - module.Backward(std::move(input2), std::move(target2), std::move(output)); + module.Backward(input2, target2, output); for (size_t i = 0; i < 8; ++i) { double el = output.at(0, i); @@ -222,25 +253,25 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) // the calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(std::move(input1), std::move(target1)); + double error1 = module.Forward(input1, target1); double expected = 0.97407699; // Value computed using tensorflow. BOOST_REQUIRE_SMALL(error1 / input1.n_elem - expected, 1e-7); input2 = arma::mat("1 2 3 4 5"); target2 = arma::mat("0 0 1 0 1"); - double error2 = module.Forward(std::move(input2), std::move(target2)); + double error2 = module.Forward(input2, target2); expected = 1.5027283; BOOST_REQUIRE_SMALL(error2 / input2.n_elem - expected, 1e-6); input3 = arma::mat("0 -1 -1 0 -1 0 0 -1"); target3 = arma::mat("0 -1 -1 0 -1 0 0 -1"); - double error3 = module.Forward(std::move(input3), std::move(target3)); + double error3 = module.Forward(input3, target3); expected = 0.00320443; BOOST_REQUIRE_SMALL(error3 / input3.n_elem - expected, 1e-6); // Test the Backward function. - module.Backward(std::move(input1), std::move(target1), std::move(output)); + module.Backward(input1, target1, output); expected = 0.62245929; for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expected, 1e-5); @@ -249,13 +280,13 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) expectedOutput = arma::mat( "0.7310586 0.88079709 -0.04742587 0.98201376 -0.00669285"); - module.Backward(std::move(input2), std::move(target2), std::move(output)); + module.Backward(input2, target2, output); for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols); - module.Backward(std::move(input3), std::move(target3), std::move(output)); + module.Backward(input3, target3, output); expectedOutput = arma::mat("0.5 1.2689414"); for (size_t i = 0; i < 8; ++i) { @@ -281,18 +312,18 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(std::move(input1), std::move(target1)); + double error1 = module.Forward(input1, target1); double expected = 0.0; BOOST_REQUIRE_SMALL(error1 / input1.n_elem - expected, 1e-7); input2 = arma::mat("1 2 3 4 5"); target2 = arma::mat("1 0 1 0 1"); - double error2 = module.Forward(std::move(input2), std::move(target2)); + double error2 = module.Forward(input2, target2); expected = -1.8; BOOST_REQUIRE_SMALL(error2 / input2.n_elem - expected, 1e-6); // Test the Backward function. - module.Backward(std::move(input1), std::move(target1), std::move(output)); + module.Backward(input1, target1, output); expected = 0.0; for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expected, 1e-5); @@ -300,7 +331,7 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); expectedOutput = arma::mat("-1 0 -1 0 -1"); - module.Backward(std::move(input2), std::move(target2), std::move(output)); + module.Backward(input2, target2, output); for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); @@ -405,16 +436,16 @@ BOOST_AUTO_TEST_CASE(DiceLossTest) // Test the Forward function. Loss should be 0 if input = target. input1 = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(std::move(input1), std::move(target)); + loss = module.Forward(input1, target); BOOST_REQUIRE_SMALL(loss, 0.00001); // Test the Forward function. Loss should be 0.185185185. input2 = arma::ones(10, 1) * 0.5; - loss = module.Forward(std::move(input2), std::move(target)); + loss = module.Forward(input2, target); BOOST_REQUIRE_CLOSE(loss, 0.185185185, 0.00001); // Test the Backward function for input = target. - module.Backward(std::move(input1), std::move(target), std::move(output)); + module.Backward(input1, target, output); for (double el : output) { // For input = target we should get 0.0 everywhere. @@ -424,7 +455,7 @@ BOOST_AUTO_TEST_CASE(DiceLossTest) BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); // Test the Backward function. - module.Backward(std::move(input2), std::move(target), std::move(output)); + module.Backward(input2, target, output); for (double el : output) { // For the 0.5 constant vector we should get -0.0877914951989026 everywhere. @@ -446,11 +477,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) // the manually calculated result. input = arma::mat("1.0 0.0 1.0 -1.0 -1.0 0.0 -1.0 0.0"); target = arma::zeros(1, 8); - double error = module.Forward(std::move(input), std::move(target)); + double error = module.Forward(input, target); BOOST_REQUIRE_EQUAL(error, 0.125); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); // We should get a vector with -1 everywhere. for (double el : output) { @@ -462,11 +493,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(std::move(input), std::move(target)); + error = module.Forward(input, target); BOOST_REQUIRE_EQUAL(error, 1.0); // Test the Backward function on a single input. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); // Test whether the output is negative. BOOST_REQUIRE_EQUAL(arma::accu(output), -1); BOOST_REQUIRE_EQUAL(output.n_elem, 1); @@ -484,11 +515,11 @@ BOOST_AUTO_TEST_CASE(LogCoshLossTest) // Test the Forward function. Loss should be 0 if input = target. input = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input, target); BOOST_REQUIRE_EQUAL(loss, 0); // Test the Backward function for input = target. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); for (double el : output) { // For input = target we should get 0.0 everywhere. @@ -501,13 +532,183 @@ BOOST_AUTO_TEST_CASE(LogCoshLossTest) // Test the Forward function. Loss should be 0.546621. input = arma::mat("1 2 3 4 5"); target = arma::mat("1 2.4 3.4 4.2 5.5"); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input, target); BOOST_REQUIRE_CLOSE(loss, 0.546621, 1e-3); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input, target, output); BOOST_REQUIRE_CLOSE(arma::accu(output), 2.46962, 1e-3); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); } + +/** + * Simple test for the Hinge Embedding loss function. + */ +BOOST_AUTO_TEST_CASE(HingeEmbeddingLossTest) +{ + arma::mat input, target, output; + double loss; + HingeEmbeddingLoss<> module; + + // Test the Forward function. Loss should be 0 if input = target. + input = arma::ones(10, 1); + target = arma::ones(10, 1); + loss = module.Forward(input, target); + BOOST_REQUIRE_EQUAL(loss, 0); + + // Test the Backward function for input = target. + module.Backward(input, target, output); + for (double el : output) + { + // For input = target we should get 0.0 everywhere. + BOOST_REQUIRE_CLOSE(el, 0.0, 1e-5); + } + + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + + // Test the Forward function. Loss should be 0.84. + input = arma::mat("0.1 0.8 0.6 0.0 0.5"); + target = arma::mat("0 1.0 1.0 0 0"); + loss = module.Forward(input, target); + BOOST_REQUIRE_CLOSE(loss, 0.84, 1e-3); + + // Test the Backward function. + module.Backward(input, target, output); + BOOST_REQUIRE_CLOSE(arma::accu(output), -2, 1e-3); + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); +} + +/** + * Simple test for the Cosine Embedding loss function. + */ +BOOST_AUTO_TEST_CASE(CosineEmbeddingLossTest) +{ + arma::mat input1, input2, y, output; + double loss; + CosineEmbeddingLoss<> module; + + // Test the Forward function. Loss should be 0 if input1 = input2 and y = 1. + input1 = arma::mat(1, 10); + input2 = arma::mat(1, 10); + input1.ones(); + input2.ones(); + y = arma::mat(1, 1); + y.ones(); + loss = module.Forward(input1, input1); + BOOST_REQUIRE_SMALL(loss, 1e-6); + + // Test the Backward function. + module.Backward(input1, input1, output); + BOOST_REQUIRE_SMALL(arma::accu(output), 1e-6); + + // Check for dissimilarity. + module.Similarity() = false; + loss = module.Forward(input1, input1); + BOOST_REQUIRE_CLOSE(loss, 1.0, 1e-4); + + // Test the Backward function. + module.Backward(input1, input1, output); + BOOST_REQUIRE_SMALL(arma::accu(output), 1e-6); + + input1 = arma::mat(3, 2); + input2 = arma::mat(3, 2); + input1.fill(1); + input1(4) = 2; + input2.fill(1); + input2(0) = 2; + input2(1) = 2; + input2(2) = 2; + loss = module.Forward(input1, input2); + // Calculated using torch.nn.CosineEmbeddingLoss(). + BOOST_REQUIRE_CLOSE(loss, 2.897367, 1e-3); + + // Test the Backward function. + module.Backward(input1, input2, output); + BOOST_REQUIRE_CLOSE(arma::accu(output), 0.06324556, 1e-3); + + // Check for correctness for cube. + CosineEmbeddingLoss<> module2(0.5, true); + + arma::cube input3(3, 2, 2); + arma::cube input4(3, 2, 2); + input3.fill(1); + input4.fill(1); + input3(0) = 2; + input3(1) = 2; + input3(4) = 2; + input3(6) = 2; + input3(8) = 2; + input3(10) = 2; + input4(2) = 2; + input4(9) = 2; + input4(11) = 2; + loss = module2.Forward(input3, input4); + // Calculated using torch.nn.CosineEmbeddingLoss(). + BOOST_REQUIRE_CLOSE(loss, 0.55395, 1e-3); + + // Test the Backward function. + module2.Backward(input3, input4, output); + BOOST_REQUIRE_CLOSE(arma::accu(output), -0.36649111, 1e-3); + + // Check Output for mean type of reduction. + CosineEmbeddingLoss<> module3(0.0, true, true); + loss = module3.Forward(input3, input4); + BOOST_REQUIRE_CLOSE(loss, 0.092325, 1e-3); + + // Check correctness for cube. + module3.Similarity() = false; + loss = module3.Forward(input3, input4); + BOOST_REQUIRE_CLOSE(loss, 0.90767498236, 1e-3); + + // Test the Backward function. + module3.Backward(input3, input4, output); + BOOST_REQUIRE_CLOSE(arma::accu(output), 0.36649111, 1e-4); +} + +/* + * Simple test for the Margin Ranking Loss function. + */ +BOOST_AUTO_TEST_CASE(MarginRankingLossTest) +{ + arma::mat input, input1, input2, target, output; + MarginRankingLoss<> module; + + // Test the Forward function on a user generator input and compare it against + // the manually calculated result. + input1 = arma::mat("1 2 5 7 -1 -3"); + input2 = arma::mat("-1 3 -4 11 3 -3"); + input = arma::join_cols(input1, input2); + target = arma::mat("1 -1 -1 1 -1 1"); + double error = module.Forward(input, target); + // Computed using torch.nn.functional.margin_ranking_loss() + BOOST_REQUIRE_CLOSE(error, 2.66667, 1e-3); + + // Test the Backward function. + module.Backward(input, target, output); + + CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 " + "0.000000 -0.000000"), 1e-3); + BOOST_REQUIRE_EQUAL(output.n_rows, target.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, target.n_cols); + + // Test the error function on another input. + input1 = arma::mat("0.4287 -1.6208 -1.5006 -0.4473 1.5208 -4.5184 9.3574 " + "-4.8090 4.3455 5.2070"); + input2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " + "-9.4886 2.2755 8.4951"); + input = arma::join_cols(input1, input2); + target = arma::mat("1 1 -1 1 -1 1 1 1 -1 1"); + error = module.Forward(input, target); + BOOST_REQUIRE_CLOSE(error, 3.03530, 1e-3); + + // Test the Backward function on the second input. + module.Backward(input, target, output); + + CheckMatrices(output, arma::mat("0.000000 0.000000 0.091240 0.000000 " + "-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/fastmks_test.cpp b/src/mlpack/tests/main_tests/fastmks_test.cpp index 6546e93e86..1d81551038 100644 --- a/src/mlpack/tests/main_tests/fastmks_test.cpp +++ b/src/mlpack/tests/main_tests/fastmks_test.cpp @@ -397,7 +397,7 @@ BOOST_AUTO_TEST_CASE(FastMKSBaseTest) SetInputParam("base", 0.0); // Invalid. Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::invalid_argument); + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; } @@ -464,6 +464,9 @@ BOOST_AUTO_TEST_CASE(FastMKSKernelTest) CLI::GetSingleton().Parameters()["reference"].wasPassed = false; CLI::GetSingleton().Parameters()["query"].wasPassed = false; CLI::GetSingleton().Parameters()["kernel"].wasPassed = false; + + if (i != nofkerneltypes - 1) + bindings::tests::CleanMemory(); } } diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 25ab918eb3..c0b8026aed 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -249,6 +249,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoiseTest) GMM* gmm1 = CLI::GetParam("output_model"); BOOST_REQUIRE(CheckDifferent(gmm, gmm1)); + + delete gmm; } // Ensure that Trials affects the final result. @@ -273,7 +275,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) mlpackMain(); - GMM* gmm = std::move(CLI::GetParam("output_model")); + GMM* gmm = CLI::GetParam("output_model"); ResetGmmTrainSetting(); @@ -290,10 +292,12 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) GMM* gmm1 = CLI::GetParam("output_model"); success = CheckDifferent(gmm, gmm1); + + delete gmm; + if (success) break; - delete gmm; bindings::tests::CleanMemory(); } @@ -332,6 +336,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainDiffMaxIterationsTest) GMM* gmm1 = CLI::GetParam("output_model"); BOOST_REQUIRE(CheckDifferent(gmm, gmm1)); + + delete gmm; } // Ensure that the maximum number of k-means iterations affects the result. @@ -375,10 +381,13 @@ BOOST_AUTO_TEST_CASE(GmmTrainDiffKmeansMaxIterationsTest) ResetGmmTrainSetting(); success = CheckDifferent(gmm, gmm1); + + delete gmm; + delete gmm1; + if (success) break; - delete gmm; bindings::tests::CleanMemory(); } @@ -419,6 +428,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) GMM* gmm1 = CLI::GetParam("output_model"); BOOST_REQUIRE(CheckDifferent(gmm, gmm1)); + + delete gmm; } // Ensure that Sampling affects the final result when refined_start is true. @@ -455,6 +466,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) GMM* gmm1 = CLI::GetParam("output_model"); BOOST_REQUIRE(CheckDifferent(gmm, gmm1)); + + delete gmm; } // Ensure that tolerance affects the final result. @@ -487,6 +500,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) GMM* gmm1 = CLI::GetParam("output_model"); BOOST_REQUIRE(CheckDifferent(gmm, gmm1)); + + delete gmm; } // Ensure that saved model can be used again. diff --git a/src/mlpack/tests/main_tests/image_converter_test.cpp b/src/mlpack/tests/main_tests/image_converter_test.cpp new file mode 100644 index 0000000000..cd77cbaf66 --- /dev/null +++ b/src/mlpack/tests/main_tests/image_converter_test.cpp @@ -0,0 +1,185 @@ +/** + * @file image_converter_test.cpp + * @author Jeffin Sam + * + * Test mlpackMain() of load_save_image_main.cpp. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#define BINDING_TYPE BINDING_TYPE_TEST + +#include +static const std::string testName = "ImageConverter"; + +#include +#include + +#include "test_helper.hpp" +#include +#include "../test_tools.hpp" + +using namespace mlpack; + +struct ImageConverterTestFixture +{ + public: + ImageConverterTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } + + ~ImageConverterTestFixture() + { + // Clear the settings. + remove("test_image777.png"); + remove("test_image999.png"); + bindings::tests::CleanMemory(); + CLI::ClearSettings(); + } +}; + +BOOST_FIXTURE_TEST_SUITE(ImageConverterMainTest, + ImageConverterTestFixture); + +BOOST_AUTO_TEST_CASE(LoadImageTest) +{ + SetInputParam>("input", {"test_image.png", "test_image.png"}); + + mlpackMain(); + arma::mat output = CLI::GetParam("output"); + // width * height * channels. + BOOST_REQUIRE_EQUAL(output.n_rows, 50 * 50 * 3); + BOOST_REQUIRE_EQUAL(output.n_cols, 2); +} + +BOOST_AUTO_TEST_CASE(SaveImageTest) +{ + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("height", 5); + SetInputParam("width", 5); + SetInputParam("channels", 3); + SetInputParam("save", true); + SetInputParam("dataset", testimage); + mlpackMain(); + + CLI::ClearSettings(); + CLI::RestoreSettings(testName); + + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("height", 5); + SetInputParam("width", 5); + SetInputParam("channels", 3); + + mlpackMain(); + arma::mat output = CLI::GetParam("output"); + BOOST_REQUIRE_EQUAL(output.n_rows, 5 * 5 * 3); + BOOST_REQUIRE_EQUAL(output.n_cols, 2); + for (size_t i = 0; i < output.n_elem; ++i) + BOOST_REQUIRE_CLOSE(testimage[i], output[i], 1e-5); +} + +/** + * Check whether binding throws error if height, width or channel are not + * specified. + */ +BOOST_AUTO_TEST_CASE(IncompleteTest) +{ + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("save", true); + SetInputParam("height", 50); + SetInputParam("width", 50); + SetInputParam("dataset", testimage); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * Check for invalid height values. + */ +BOOST_AUTO_TEST_CASE(InvalidInputTest) +{ + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("save", true); + SetInputParam("dataset", testimage); + + SetInputParam("height", -50); + SetInputParam("width", 50); + SetInputParam("channels", 3); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * Check for invalid width values. + */ +BOOST_AUTO_TEST_CASE(InvalidWidthTest) +{ + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("save", true); + SetInputParam("dataset", testimage); + SetInputParam("height", 50); + SetInputParam("width", -50); + SetInputParam("channels", 3); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * Check for invalid channel values. + */ +BOOST_AUTO_TEST_CASE(InvalidChannelTest) +{ + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("save", true); + SetInputParam("dataset", testimage); + SetInputParam("height", 50); + SetInputParam("width", 50); + SetInputParam("channels", -1); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * Check for invalid input values. + */ +BOOST_AUTO_TEST_CASE(EmptyInputTest) +{ + SetInputParam>("input", {}); + SetInputParam("height", 50); + SetInputParam("width", 50); + SetInputParam("channels", 50); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/kde_test.cpp b/src/mlpack/tests/main_tests/kde_test.cpp index b8666cc165..69f38c60cf 100644 --- a/src/mlpack/tests/main_tests/kde_test.cpp +++ b/src/mlpack/tests/main_tests/kde_test.cpp @@ -30,14 +30,15 @@ struct KDETestFixture public: KDETestFixture() { - // Cache in the options for this program. - CLI::RestoreSettings(testName); + // Cache in the options for this program. + CLI::RestoreSettings(testName); } ~KDETestFixture() { - // Clear the settings. - CLI::ClearSettings(); + // Clear the settings. + bindings::tests::CleanMemory(); + CLI::ClearSettings(); } }; @@ -545,6 +546,8 @@ BOOST_AUTO_TEST_CASE(KDEMainMonteCarloFlag) mlpackMain(); estimations1 = std::move(CLI::GetParam("predictions")); + delete CLI::GetParam("output_model"); + // Compute estimations 2. SetInputParam("reference", reference); SetInputParam("query", query); diff --git a/src/mlpack/tests/main_tests/knn_test.cpp b/src/mlpack/tests/main_tests/knn_test.cpp index 18f91af45c..02b8c2e75d 100644 --- a/src/mlpack/tests/main_tests/knn_test.cpp +++ b/src/mlpack/tests/main_tests/knn_test.cpp @@ -683,6 +683,7 @@ BOOST_AUTO_TEST_CASE(KNNDifferentLeafSizes) BOOST_CHECK_EQUAL(output_model->LeafSize(), (int) 1); BOOST_CHECK_EQUAL(CLI::GetParam("output_model")->LeafSize(), (int) 10); - } + delete output_model; +} BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/linear_svm_test.cpp b/src/mlpack/tests/main_tests/linear_svm_test.cpp index 057a33d1e5..c0d1c5f136 100644 --- a/src/mlpack/tests/main_tests/linear_svm_test.cpp +++ b/src/mlpack/tests/main_tests/linear_svm_test.cpp @@ -414,7 +414,7 @@ BOOST_AUTO_TEST_CASE(LinearSVMNonNegativeEpochsTest) } /** - * Ensuring that number classes must not be zero. + * Ensuring that number classes must not be one. */ BOOST_AUTO_TEST_CASE(LinearSVMZeroNumberOfClassesTest) { @@ -429,7 +429,7 @@ BOOST_AUTO_TEST_CASE(LinearSVMZeroNumberOfClassesTest) SetInputParam("training", std::move(trainData)); SetInputParam("labels", std::move(trainLabels)); - // Number of classes for optimizer is zero. + // Number of classes for optimizer is only one. // It should throw a invalid_argument error. Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::invalid_argument); diff --git a/src/mlpack/tests/main_tests/preprocess_scale_test.cpp b/src/mlpack/tests/main_tests/preprocess_scale_test.cpp index debf848883..934542584b 100644 --- a/src/mlpack/tests/main_tests/preprocess_scale_test.cpp +++ b/src/mlpack/tests/main_tests/preprocess_scale_test.cpp @@ -57,16 +57,18 @@ BOOST_AUTO_TEST_CASE(TwoScalerTest) SetInputParam("scaler_method", method); mlpackMain(); - arma::mat max_abs_scaler_output = CLI::GetParam("output"); + arma::mat maxAbsScalerOutput = CLI::GetParam("output"); + + bindings::tests::CleanMemory(); method = "standard_scaler"; SetInputParam("input", dataset); SetInputParam("scaler_method", std::move(method)); mlpackMain(); - arma::mat standard_scaler_output = CLI::GetParam("output"); + arma::mat standardScalerOutput = CLI::GetParam("output"); - CheckMatricesNotEqual(standard_scaler_output, max_abs_scaler_output); + CheckMatricesNotEqual(standardScalerOutput, maxAbsScalerOutput); } /** @@ -83,6 +85,8 @@ BOOST_AUTO_TEST_CASE(TwoOptionTest) mlpackMain(); arma::mat output = CLI::GetParam("output"); + bindings::tests::CleanMemory(); + SetInputParam("input", dataset); SetInputParam("scaler_method", std::move(method)); SetInputParam("min_value", 2); @@ -107,6 +111,8 @@ BOOST_AUTO_TEST_CASE(UnrelatedOptionTest) mlpackMain(); arma::mat scaled = CLI::GetParam("output"); + bindings::tests::CleanMemory(); + SetInputParam("input", dataset); SetInputParam("scaler_method", std::move(method)); SetInputParam("min_value", 2); @@ -177,6 +183,8 @@ BOOST_AUTO_TEST_CASE(EpsilonTest) mlpackMain(); arma::mat scaled = CLI::GetParam("output"); + bindings::tests::CleanMemory(); + SetInputParam("scaler_method", std::move(method)); SetInputParam("input", dataset); SetInputParam("epsilon", 1.0); diff --git a/src/mlpack/tests/main_tests/range_search_test.cpp b/src/mlpack/tests/main_tests/range_search_test.cpp index 5673e1ef09..e28b14a4a3 100644 --- a/src/mlpack/tests/main_tests/range_search_test.cpp +++ b/src/mlpack/tests/main_tests/range_search_test.cpp @@ -278,7 +278,7 @@ BOOST_AUTO_TEST_CASE(ModelCheck) neighbors = ReadData(neighborsFile); distances = ReadData(distanceFile); - RSModel* outputModel = move(CLI::GetParam("output_model")); + RSModel* outputModel = CLI::GetParam("output_model"); CLI::GetSingleton().Parameters()["reference"].wasPassed = false; SetInputParam("input_model", outputModel); @@ -293,7 +293,7 @@ BOOST_AUTO_TEST_CASE(ModelCheck) CheckMatrices(distances, distancetemp); BOOST_REQUIRE_EQUAL(ModelToString(outputModel), - ModelToString(CLI::GetParam("output_model"))); + ModelToString(CLI::GetParam("output_model"))); remove(neighborsFile.c_str()); remove(distanceFile.c_str()); @@ -351,8 +351,13 @@ BOOST_AUTO_TEST_CASE(LeafValueTesting) BOOST_REQUIRE_NE(ModelToString(outputModel1), ModelToString(CLI::GetParam("output_model"))); + + if (i != leafSizes.size() - 1) + delete CLI::GetParam("output_model"); } + delete outputModel1; + remove(neighborsFile.c_str()); remove(distanceFile.c_str()); } @@ -419,8 +424,13 @@ BOOST_AUTO_TEST_CASE(TreeTypeTesting) CheckMatrices(distances, distancestemp); BOOST_REQUIRE_NE(ModelToString(outputModel1), ModelToString(CLI::GetParam("output_model"))); + + if (i != trees.size() - 1) + delete CLI::GetParam("output_model"); } + delete outputModel1; + remove(neighborsFile.c_str()); remove(distanceFile.c_str()); } @@ -463,6 +473,8 @@ BOOST_AUTO_TEST_CASE(RandomBasisTesting) BOOST_REQUIRE_NE(ModelToString(outputModel), ModelToString(CLI::GetParam("output_model"))); + delete outputModel; + remove(neighborsFile.c_str()); remove(distanceFile.c_str()); } @@ -515,6 +527,8 @@ BOOST_AUTO_TEST_CASE(NaiveModeTest) BOOST_REQUIRE_NE(ModelToString(outputModel), ModelToString(CLI::GetParam("output_model"))); + delete outputModel; + remove(neighborsFile.c_str()); remove(distanceFile.c_str()); } @@ -566,6 +580,8 @@ BOOST_AUTO_TEST_CASE(SingleModeTest) BOOST_REQUIRE_NE(ModelToString(outputModel), ModelToString(CLI::GetParam("output_model"))); + delete outputModel; + remove(neighborsFile.c_str()); remove(distanceFile.c_str()); } diff --git a/src/mlpack/tests/mlpack_test.cpp b/src/mlpack/tests/mlpack_test.cpp index 1518d0afd9..0d7cdae5b7 100644 --- a/src/mlpack/tests/mlpack_test.cpp +++ b/src/mlpack/tests/mlpack_test.cpp @@ -14,11 +14,6 @@ #include -// We only need to do this for old Boost versions. -#if BOOST_VERSION < 103600 - #define BOOST_AUTO_TEST_MAIN -#endif - #if BOOST_VERSION >= 105900 #include #include diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index e7c30b38f8..95a26f40ad 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN) config.Discount() = 0.9; config.TargetNetworkSyncInterval() = 100; config.ExplorationSteps() = 100; - config.DoubleQLearning() = false; + config.DoubleQLearning() = true; config.StepLimit() = 200; // Set up the DQN agent. @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN) averageReturn(episodeReturn); /** - * Reaching running average return 35 is enough to show it works. + * Reaching running average return 40 is enough to show it works. * For the speed of the test case, I didn't set high criterion. */ Log::Debug << "Average return: " << averageReturn.mean() diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index ddf328ecc0..9d178f12e0 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -114,7 +114,7 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) BOOST_TEST_CHECKPOINT("Training over"); arma::cube prediction; model.Predict(input, prediction); - BOOST_TEST_CHECKPOINT("Predicion over"); + BOOST_TEST_CHECKPOINT("Prediction over"); size_t error = 0; for (size_t i = 0; i < prediction.n_cols; ++i) @@ -825,7 +825,7 @@ void DistractedSequenceRecallTestNetwork( inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, trainInput.at(0, j).n_elem / inputSize, false, true); labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), outputSize, 1, - trainInput.at(0, j).n_elem / outputSize, false, true); + trainLabels.at(0, j).n_elem / outputSize, false, true); model.Train(inputTemp, labelsTemp, opt); } diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 9fb4e188dd..31a0f22713 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include "test_tools.hpp" @@ -54,6 +56,25 @@ static vector stringEncodingUtf8Input = { "\xE2\x93\x82\xE2\x93\x81\xE2\x93\x85\xE2\x92\xB6\xE2\x92\xB8\xE2\x93\x80" }; +/** + * Check the values of two 2D vectors. + */ +template +void CheckVectors(const vector>& a, + const vector>& b, + const ValueType tolerance = 1e-5) +{ + BOOST_REQUIRE_EQUAL(a.size(), b.size()); + + for (size_t i = 0; i < a.size(); i++) + { + BOOST_REQUIRE_EQUAL(a[i].size(), b[i].size()); + + for (size_t j = 0; j < a[i].size(); j++) + BOOST_REQUIRE_CLOSE(a[i][j], b[i][j], tolerance); + } +} + /** * Test the dictionary encoding algorithm. */ @@ -88,7 +109,7 @@ BOOST_AUTO_TEST_CASE(DictionaryEncodingTest) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; - CheckMatrices(output, expected); + CheckMatrices(output, expected.t()); } /** @@ -122,7 +143,7 @@ BOOST_AUTO_TEST_CASE(UnicodeDictionaryEncodingTest) { 5, 2, 3, 5, 4 } }; - CheckMatrices(output, expected); + CheckMatrices(output, expected.t()); } /** @@ -222,8 +243,8 @@ BOOST_AUTO_TEST_CASE(SplitByAnyOfTokenizerUnicodeTest) } /** -* Test the CharExtract tokenizer. -*/ + * Test the CharExtract tokenizer. + */ BOOST_AUTO_TEST_CASE(DictionaryEncodingIndividualCharactersTest) { vector input = { @@ -242,7 +263,7 @@ BOOST_AUTO_TEST_CASE(DictionaryEncodingIndividualCharactersTest) { 2, 4, 3, 2, 4, 3, 5 }, { 1, 2, 4, 0, 0, 0, 0 } }; - CheckMatrices(output, target); + CheckMatrices(output, target.t()); } /** @@ -520,5 +541,889 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) CheckMatrices(output, xmlOutput, textOutput, binaryOutput); } +/** + * Test the Bag of Words encoding algorithm. + */ +BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + arma::mat output; + BagOfWordsEncoding encoder; + SplitByAnyOf tokenizer(" ,."); + + encoder.Encode(stringEncodingInput, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that each token has a unique label. + std::unordered_map keysCount; + + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + +/* The expected values were obtained by the following Python script: + + from sklearn.feature_extraction.text import CountVectorizer + from collections import OrderedDict + import re + + string_encoding_input = [ + "mlpack is an intuitive, fast, and flexible C++ machine learning library " + "with bindings to other languages. ", + "It is meant to be a machine learning analog to LAPACK, and aims to " + "implement a wide array of machine learning methods and functions " + "as a \"swiss army knife\" for machine learning researchers.", + "In addition to its powerful C++ interface, mlpack also provides " + "command-line programs and Python bindings." + ] + + dictionary = OrderedDict() + + count = 0 + for line in string_encoding_input: + for word in re.split(' |,|\.', line): + if word and (not (word in dictionary)): + dictionary[word] = count + count += 1 + + def tokenizer(line): + return re.split(' |,|\.', line) + + vectorizer = CountVectorizer(strip_accents=False, lowercase=False, + preprocessor=None, tokenizer=tokenizer, stop_words=None, + vocabulary=dictionary, binary=False) + + X = vectorizer.fit_transform(string_encoding_input) + + for row in X.toarray(): + print("{ " + ", ".join(map(str, row)) + " },") +*/ + + arma::mat expected = { + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 0, 0, 0, 2, 0, 0, 3, 3, 0, 0, 0, 3, 0, 0, 1, 1, 1, 3, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + }; + + CheckMatrices(output, expected.t()); +} + +/** + * Test the Bag of Words encoding algorithm. The output is saved into a vector. + */ +BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + vector> output; + BagOfWordsEncoding encoder( + (BagOfWordsEncodingPolicy())); + SplitByAnyOf tokenizer(" ,."); + + encoder.Encode(stringEncodingInput, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that each token has a unique label. + std::unordered_map keysCount; + + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + + /* The expected values were obtained by the same script as in + BagOfWordsEncodingTest. */ + vector> expected = { + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 0, 0, 0, 2, 0, 0, 3, 3, 0, 0, 0, 3, 0, 0, 1, 1, 1, 3, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + }; + + BOOST_REQUIRE(output == expected); +} + +/** + * Test the Bag of Words algorithm for individual characters. + */ +BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + BagOfWordsEncoding encoder; + + encoder.Encode(input, output, CharExtract()); + + arma::mat target = { + { 1, 2, 2, 0, 0 }, + { 0, 2, 2, 2, 1 }, + { 1, 1, 0, 1, 0 } + }; + + CheckMatrices(output, target.t()); +} + +/** + * Test the Bag of Words encoding algorithm in case of individual + * characters encoding. The output type is vector>. + */ +BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + vector> output; + BagOfWordsEncoding encoder; + + encoder.Encode(input, output, CharExtract()); + + vector> expected = { + { 1, 2, 2, 0, 0 }, + { 0, 2, 2, 2, 1 }, + { 1, 1, 0, 1, 0 } + }; + + BOOST_REQUIRE(output == expected); +} + +/** + * Test the Tf-Idf encoding algorithm with the raw count term frequency type + * and the smooth inverse document frequency type. These parameters are + * the default ones. + */ +BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + arma::mat output; + TfIdfEncoding encoder; + SplitByAnyOf tokenizer(" ,."); + + encoder.Encode(stringEncodingInput, output, tokenizer); + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that each token has a unique label. + std::unordered_map keysCount; + + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + + /* The expected values were obtained by the following Python script: + + from sklearn.feature_extraction.text import TfidfVectorizer + from collections import OrderedDict + import re + + string_encoding_input = [ + "mlpack is an intuitive, fast, and flexible C++ machine learning library " + "with bindings to other languages. ", + "It is meant to be a machine learning analog to LAPACK, and aims to " + "implement a wide array of machine learning methods and functions " + "as a \"swiss army knife\" for machine learning researchers.", + "In addition to its powerful C++ interface, mlpack also provides " + "command-line programs and Python bindings." + ] + + smooth_idf = True + tf_type = 'raw_count' + + dictionary = OrderedDict() + + count = 0 + for line in string_encoding_input: + for word in re.split(' |,|\.', line): + if word and (not (word in dictionary)): + dictionary[word] = count + count += 1 + + def tokenizer(line): + return re.split(' |,|\.', line) + + if tf_type == 'raw_count': + binary = False + sublinear_tf = False + elif tf_type == 'binary': + binary = True + sublinear_tf = False + elif tf_type == 'sublinear_tf': + binary = False + sublinear_tf = True + + vectorizer = TfidfVectorizer(strip_accents=False, lowercase=False, + preprocessor=None, tokenizer=tokenizer, stop_words=None, + vocabulary=dictionary, binary=binary, norm=None, smooth_idf=smooth_idf, + sublinear_tf=sublinear_tf) + + X = vectorizer.fit_transform(string_encoding_input) + + def format_result(value): + if value == int(value): + return str(int(value)) + else: + return "{0:.8f}".format(value) + + for row in X.toarray(): + print("{ " + ", ".join(map(format_result, row)) + " },") + */ + arma::mat expected = { + { 1.28768207, 1.28768207, 1.69314718, 1.69314718, 1.69314718, 1, 1.69314718, + 1.28768207, 1.28768207, 1.28768207, 1.69314718, 1.69314718, 1.28768207, 1, + 1.69314718, 1.69314718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1.28768207, 0, 0, 0, 2, 0, 0, 3.86304622, 3.86304622, 0, 0, 0, 3, 0, + 0, 1.69314718, 1.69314718, 1.69314718, 5.07944154, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207, 0, 0, 0, 0, 1, 0, 1.28768207, 0, 0, 0, 0, 1.28768207, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718 } + }; + + CheckMatrices(output, expected.t(), 1e-6); +} + +/** + * Test the Tf-Idf encoding algorithm with the raw count term frequency type + * and the smooth inverse document frequency type. These parameters are + * the default ones. The output type is vector>. + */ +BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + vector> output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy())); + SplitByAnyOf tokenizer(" ,."); + + encoder.Encode(stringEncodingInput, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that each token has a unique label. + std::unordered_map keysCount; + + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + + /* The expected values were obtained by the same script as in + RawCountSmoothIdfEncodingTest. */ + vector> expected = { + { 1.28768207, 1.28768207, 1.69314718, 1.69314718, 1.69314718, 1, 1.69314718, + 1.28768207, 1.28768207, 1.28768207, 1.69314718, 1.69314718, 1.28768207, 1, + 1.69314718, 1.69314718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1.28768207, 0, 0, 0, 2, 0, 0, 3.86304622, 3.86304622, 0, 0, 0, 3, 0, + 0, 1.69314718, 1.69314718, 1.69314718, 5.07944154, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207, 0, 0, 0, 0, 1, 0, 1.28768207, 0, 0, 0, 0, 1.28768207, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718 } + }; + CheckVectors(output, expected, 1e-6); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * raw count term frequency type and the smooth inverse document frequency type. + * These parameters are the default ones. + */ +BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder; + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by the following Python script: + + from sklearn.feature_extraction.text import TfidfVectorizer + from collections import OrderedDict + import re + + input_string = [ + "GACCA", + "ABCABCD", + "GAB" + ] + + smooth_idf = True + tf_type = 'raw_count' + + dictionary = OrderedDict() + + count = 0 + for line in input_string: + for word in list(line): + if word and (not (word in dictionary)): + dictionary[word] = count + count += 1 + + def tokenizer(line): + return list(line) + + if tf_type == 'raw_count': + binary = False + sublinear_tf = False + elif tf_type == 'binary': + binary = True + sublinear_tf = False + elif tf_type == 'sublinear_tf': + binary = False + sublinear_tf = True + + vectorizer = TfidfVectorizer(strip_accents=False, lowercase=False, + preprocessor=None, tokenizer=tokenizer, stop_words=None, + vocabulary=dictionary, binary=binary, norm=None, smooth_idf=smooth_idf, + sublinear_tf=sublinear_tf) + + X = vectorizer.fit_transform(input_string) + + def format_result(value): + if value == int(value): + return str(int(value)) + else: + return "{0:.14f}".format(value) + + for row in X.toarray(): + print("{ " + ", ".join(map(format_result, row)) + " },") + */ + arma::mat target = { + { 1.28768207245178, 2, 2.57536414490356, 0, 0 }, + { 0, 2, 2.57536414490356, 2.57536414490356, 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } + }; + + CheckMatrices(output, target.t(), 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * raw count term frequency type and the smooth inverse document frequency type. + * These parameters are the default ones. The output type is + * vector>. + */ +BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + vector> output; + TfIdfEncoding encoder; + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. */ + vector> expected = { + { 1.28768207245178, 2, 2.57536414490356, 0, 0 }, + { 0, 2, 2.57536414490356, 2.57536414490356, 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } + }; + + CheckVectors(output, expected, 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm with the raw count term frequency type + * and the non-smooth inverse document frequency type. + */ +BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + arma::mat output; + TfIdfEncoding encoder( + TfIdfEncodingPolicy(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false)); + SplitByAnyOf tokenizer(" ,."); + + encoder.Encode(stringEncodingInput, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that each token has a unique label. + std::unordered_map keysCount; + + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingTest. The only difference is smooth_idf equals + False. */ + arma::mat expected = { + { 1.40546511, 1.40546511, 2.09861229, 2.09861229, 2.09861229, 1, 2.09861229, + 1.40546511, 1.40546511, 1.40546511, 2.09861229, 2.09861229, 1.40546511, 1, + 2.09861229, 2.09861229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1.40546511, 0, 0, 0, 2, 0, 0, 4.21639532, 4.21639532, 0, 0, 0, 3, 0, 0, + 2.09861229, 2.09861229, 2.09861229, 6.29583687, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546511, 0, 0, 0, 0, 1, 0, 1.40546511, 0, 0, 0, 0, 1.40546511, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229 } + }; + + CheckMatrices(output, expected.t(), 1e-6); +} + +/** + * Test the Tf-Idf encoding algorithm with the raw count term frequency type + * and the non-smooth inverse document frequency type. The output type is + * vector>. + */ +BOOST_AUTO_TEST_CASE(VectorTfIdfRawCountEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + vector> output; + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); + SplitByAnyOf tokenizer(" ,."); + + encoder.Encode(stringEncodingInput, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that each token has a unique label. + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingTest. The only difference is smooth_idf equals + False. */ + vector> expected = { + { 1.40546511, 1.40546511, 2.09861229, 2.09861229, 2.09861229, 1, 2.09861229, + 1.40546511, 1.40546511, 1.40546511, 2.09861229, 2.09861229, 1.40546511, 1, + 2.09861229, 2.09861229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1.40546511, 0, 0, 0, 2, 0, 0, 4.21639532, 4.21639532, 0, 0, 0, 3, 0, 0, + 2.09861229, 2.09861229, 2.09861229, 6.29583687, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546511, 0, 0, 0, 0, 1, 0, 1.40546511, 0, 0, 0, 0, 1.40546511, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229 } + }; + CheckVectors(output, expected, 1e-6); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * raw count term frequency type and the non-smooth inverse document frequency + * type. + */ +BOOST_AUTO_TEST_CASE(RawCountTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + smooth_idf equals False. */ + arma::mat target = { + { 1.40546510810816, 2, 2.81093021621633, 0, 0 }, + { 0, 2, 2.81093021621633, 2.81093021621633, 2.09861228866811 }, + { 1.40546510810816, 1, 0, 1.40546510810816, 0 } + }; + + CheckMatrices(output, target.t(), 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * raw count term frequency type and the non-smooth inverse document frequency + * type. The output type is vector>. + */ +BOOST_AUTO_TEST_CASE(VectorRawCountTfIdfEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + vector> output; + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + smooth_idf equals False. */ + vector> expected = { + { 1.40546510810816, 2, 2.81093021621633, 0, 0 }, + { 0, 2, 2.81093021621633, 2.81093021621633, 2.09861228866811 }, + { 1.40546510810816, 1, 0, 1.40546510810816, 0 } + }; + + CheckVectors(output, expected, 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * binary term frequency type and the smooth inverse document frequency type. + */ +BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::BINARY, true); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'binary'. */ + arma::mat target = { + { 1.28768207245178, 1, 1.28768207245178, 0, 0 }, + { 0, 1, 1.28768207245178, 1.28768207245178, 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } + }; + + CheckMatrices(output, target.t(), 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * binary term frequency type and the smooth inverse document frequency type. + * The output type is vector>. + */ +BOOST_AUTO_TEST_CASE(VectorBinarySmoothIdfEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + vector> output; + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::BINARY, true); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'binary'. */ + vector> expected = { + { 1.28768207245178, 1, 1.28768207245178, 0, 0 }, + { 0, 1, 1.28768207245178, 1.28768207245178, 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } + }; + + CheckVectors(output, expected, 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * binary term frequency type and the non-smooth inverse document frequency + * type. + */ +BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::BINARY, false); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'binary' and smooth_idf equals False. */ + arma::mat target = { + { 1.40546510810816, 1, 1.40546510810816, 0, 0 }, + { 0, 1, 1.40546510810816, 1.40546510810816, 2.09861228866811 }, + { 1.40546510810816, 1, 0, 1.40546510810816, 0 } + }; + + CheckMatrices(output, target.t(), 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * sublinear term frequency type and the smooth inverse document frequency + * type. + */ +BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, true); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'sublinear_tf'. */ + arma::mat target = { + { 1.28768207245178, 1.69314718055995, 2.18023527042932, 0, 0 }, + { 0, 1.69314718055995, 2.18023527042932, 2.18023527042932, + 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } + }; + + CheckMatrices(output, target.t(), 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * sublinear term frequency type and the non-smooth inverse document frequency + * type. + */ +BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, false); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'sublinear_tf' and smooth_idf equals False. */ + arma::mat target = { + { 1.40546510810816, 1.69314718055995, 2.37965928516872, 0, 0 }, + { 0, 1.69314718055995, 2.37965928516872, 2.37965928516872, + 2.09861228866811 }, + { 1.40546510810816, 1, 0, 1.40546510810816, 0 } + }; + + CheckMatrices(output, target.t(), 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * standard term frequency type and the smooth inverse document frequency + * type. + */ +BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, true); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by the following Python script: + + from sklearn.feature_extraction.text import CountVectorizer + from sklearn.feature_extraction.text import TfidfTransformer + from collections import OrderedDict + import numpy as np + import re + + input_string = [ + "GACCA", + "ABCABCD", + "GAB" + ] + + smooth_idf = True + + dictionary = OrderedDict() + + count = 0 + for line in input_string: + for word in list(line): + if word and (not (word in dictionary)): + dictionary[word] = count + count += 1 + + def tokenizer(line): + return list(line) + + vectorizer = CountVectorizer(strip_accents=False, lowercase=False, + preprocessor=None, tokenizer=tokenizer, stop_words=None, + vocabulary=dictionary, binary=False) + + count = vectorizer.fit_transform(input_string) + + lens = np.array(list(map(len, input_string))).reshape(len(input_string), 1) + + tf = count.toarray() / lens + + transformer = TfidfTransformer(norm=None, smooth_idf=smooth_idf, + sublinear_tf=False) + + X = transformer.fit_transform(tf) + + def format_result(value): + if value == int(value): + return str(int(value)) + else: + return "{0:.16}".format(value) + + for row in X.toarray(): + print("{ " + ", ".join(map(format_result, row)) + " },") + */ + arma::mat target = { + { 0.2575364144903562, 0.4, 0.5150728289807124, 0, 0 }, + { 0, 0.2857142857142857, 0.3679091635576516, 0.3679091635576516, + 0.2418781686514208 }, + { 0.4292273574839269, 0.3333333333333333, 0, 0.4292273574839269, 0 } + }; + + CheckMatrices(output, target.t(), 1e-12); +} + +/** + * Test the Tf-Idf encoding algorithm for individual characters with the + * standard term frequency type and the non-smooth inverse document frequency + * type. + */ +BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, false); + + encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + TermFrequencySmoothIdfEncodingIndividualCharactersTest. The only difference + is smooth_idf equals False. */ + arma::mat target = { + { 0.2810930216216329, 0.4, 0.5621860432432658, 0, 0 }, + { 0, 0.2857142857142857, 0.4015614594594755, 0.4015614594594755, + 0.2998017555240157 }, + { 0.4684883693693881, 0.3333333333333333, 0, 0.4684883693693881, 0 } + }; + + CheckMatrices(output, target.t(), 1e-12); +} + +/** + * Serialization test for the Tf-Idf encoding algorithm with + * the SplitByAnyOf tokenizer. + */ +BOOST_AUTO_TEST_CASE(SplitByAnyOfTfIdfEncodingSerialization) +{ + using EncoderType = TfIdfEncoding; + + EncoderType encoder; + SplitByAnyOf tokenizer(" ,.\""); + arma::mat output; + + encoder.Encode(stringEncodingInput, output, tokenizer); + + EncoderType xmlEncoder, textEncoder, binaryEncoder; + arma::mat xmlOutput, textOutput, binaryOutput; + + SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); + + CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); + CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); + CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); + + xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); + textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); + binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); + + CheckMatrices(output, xmlOutput, textOutput, binaryOutput); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/tree_test.cpp b/src/mlpack/tests/tree_test.cpp index 22deb31a58..af8eaeb081 100644 --- a/src/mlpack/tests/tree_test.cpp +++ b/src/mlpack/tests/tree_test.cpp @@ -2129,12 +2129,15 @@ BOOST_AUTO_TEST_CASE(BinarySpaceTreeCopyConstructor) TreeType b(data); b.Begin() = 10; b.Count() = 50; + b.Left() = new TreeType(data); b.Left()->Begin() = 10; b.Left()->Count() = 30; + b.Left()->Parent() = &b; b.Right() = new TreeType(data); b.Right()->Begin() = 40; b.Right()->Count() = 20; + b.Right()->Parent() = &b; // Copy the tree. TreeType c(b); @@ -2159,6 +2162,11 @@ BOOST_AUTO_TEST_CASE(BinarySpaceTreeCopyConstructor) BOOST_REQUIRE_EQUAL(b.Right()->Left(), c.Right()->Left()); BOOST_REQUIRE_EQUAL(b.Right()->Right(), (TreeType*) NULL); BOOST_REQUIRE_EQUAL(b.Right()->Right(), c.Right()->Right()); + + // Clean memory (we built the tree by hand, so this is what we have to do + // since the destructor won't free the children's datasets). + delete &b.Left()->Dataset(); + delete &b.Right()->Dataset(); } //! Count the number of leaves under this node.