diff --git a/CMakeLists.txt b/CMakeLists.txt index 5641c5b437..ad6bf23d88 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,6 @@ option(BUILD_TESTS "Build tests." ON) option(BUILD_CLI_EXECUTABLES "Build command-line executables." ON) option(DOWNLOAD_DEPENDENCIES "Automatically download dependencies if not available." OFF) option(BUILD_GO_SHLIB "Build Go shared library." OFF) -option(BUILD_DOCS "Build doxygen documentation (if doxygen is available)." ON) # Set minimum library versions required by mlpack. # @@ -484,48 +483,6 @@ set(MLPACK_LIBRARIES ${MLPACK_LIBRARIES} ${CROSS_COMPILE_SUPPORT_LIBRARIES}) # Recurse into the rest of the project. add_subdirectory(src/mlpack) -# Make a target to generate the documentation. If Doxygen isn't installed, then -# I guess this option will just be unavailable. -if (BUILD_DOCS) - find_package(Doxygen) - if (DOXYGEN_FOUND) - if (MATHJAX) - find_package(MathJax) - if (NOT MATHJAX_FOUND) - message(STATUS "Using MathJax at the MathJax Content Delivery Network. " - "Be careful, formulas will not be shown without the internet.") - endif () - endif () - # Preprocess the Doxyfile. This is done before 'make doc'. - add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/Doxyfile - PRE_BUILD - COMMAND ${CMAKE_COMMAND} - -D DESTDIR=${CMAKE_BINARY_DIR} - -D MATHJAX="${MATHJAX}" - -D MATHJAX_FOUND="${MATHJAX_FOUND}" - -D MATHJAX_PATH="${MATHJAX_PATH}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/CMake/GenerateDoxyfile.cmake" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile" - COMMENT "Creating Doxyfile to generate Doxygen documentation" - ) - - # Generate documentation. - add_custom_target(doc - COMMAND "${DOXYGEN_EXECUTABLE}" "${CMAKE_BINARY_DIR}/Doxyfile" - DEPENDS "${CMAKE_BINARY_DIR}/Doxyfile" - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" - COMMENT "Generating API documentation with Doxygen" - ) - - install(DIRECTORY "${CMAKE_BINARY_DIR}/doc/html" - DESTINATION "${CMAKE_INSTALL_DOCDIR}" - COMPONENT doc - OPTIONAL - ) - endif () -endif() - # Create the pkg-config file, if we have pkg-config. find_package(PkgConfig) if (PKG_CONFIG_FOUND) diff --git a/Doxyfile b/Doxyfile deleted file mode 100644 index b7afc561b9..0000000000 --- a/Doxyfile +++ /dev/null @@ -1,262 +0,0 @@ -# Doxyfile 1.4.7 - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- -PROJECT_NAME = mlpack -PROJECT_NUMBER = 3.4.2 -OUTPUT_DIRECTORY = ./doc -CREATE_SUBDIRS = NO -OUTPUT_LANGUAGE = English -USE_WINDOWS_ENCODING = NO -BRIEF_MEMBER_DESC = YES -REPEAT_BRIEF = YES -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the -ALWAYS_DETAILED_SEC = YES -INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = YES -STRIP_FROM_PATH = ./ -STRIP_FROM_INC_PATH = -SHORT_NAMES = NO -JAVADOC_AUTOBRIEF = YES -MULTILINE_CPP_IS_BRIEF = NO -DETAILS_AT_TOP = YES -INHERIT_DOCS = YES -SEPARATE_MEMBER_PAGES = NO -TAB_SIZE = 2 -ALIASES = -OPTIMIZE_OUTPUT_FOR_C = NO -OPTIMIZE_OUTPUT_JAVA = NO -BUILTIN_STL_SUPPORT = NO -DISTRIBUTE_GROUP_DOC = NO -SUBGROUPING = YES -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- -EXTRACT_ALL = YES -EXTRACT_PRIVATE = NO -EXTRACT_STATIC = YES -EXTRACT_LOCAL_CLASSES = NO -EXTRACT_LOCAL_METHODS = NO -HIDE_UNDOC_MEMBERS = NO -HIDE_UNDOC_CLASSES = NO -HIDE_FRIEND_COMPOUNDS = YES -HIDE_IN_BODY_DOCS = NO -INTERNAL_DOCS = YES -CASE_SENSE_NAMES = YES -HIDE_SCOPE_NAMES = YES -SHOW_INCLUDE_FILES = NO -INLINE_INFO = YES -SORT_MEMBER_DOCS = YES -SORT_BRIEF_DOCS = YES -SORT_BY_SCOPE_NAME = YES -SORT_MEMBERS_CTORS_1ST = YES -GENERATE_TODOLIST = NO -GENERATE_TESTLIST = NO -GENERATE_BUGLIST = YES -GENERATE_DEPRECATEDLIST= NO -ENABLED_SECTIONS = -MAX_INITIALIZER_LINES = 30 -SHOW_USED_FILES = YES -SHOW_DIRECTORIES = YES -FILE_VERSION_FILTER = -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- -QUIET = NO -WARNINGS = YES -# This will be set to YES for the Jenkins doxygen check build. -WARN_AS_ERROR = NO -WARN_IF_UNDOCUMENTED = YES -WARN_IF_DOC_ERROR = YES -WARN_NO_PARAMDOC = YES -WARN_FORMAT = "$file:$line: $text" -WARN_LOGFILE = -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- -INPUT = ./src/mlpack \ - ./doc/guide \ - ./doc/tutorials \ - ./doc/policies -FILE_PATTERNS = *.hpp \ - *.cpp \ - *.txt -RECURSIVE = YES -EXCLUDE = -EXCLUDE_SYMLINKS = YES -EXCLUDE_PATTERNS = */build/* \ - */test/* \ - */arma_extend/* \ - */boost_backport/* \ - */.svn/* \ - *_impl.cc \ - *_impl.h \ - *_impl.hpp \ - *.cpp \ - *.cc \ - *_test.cpp \ - *CLI11.hpp \ - */tests/catch.hpp \ - */boost/serialization/* -EXAMPLE_PATH = -EXAMPLE_PATTERNS = * -EXAMPLE_RECURSIVE = NO -IMAGE_PATH = -INPUT_FILTER = -FILTER_PATTERNS = -FILTER_SOURCE_FILES = NO -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- -SOURCE_BROWSER = YES -INLINE_SOURCES = NO -STRIP_CODE_COMMENTS = YES -REFERENCED_BY_RELATION = YES -REFERENCES_RELATION = YES -REFERENCES_LINK_SOURCE = YES -USE_HTAGS = NO -VERBATIM_HEADERS = YES -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- -ALPHABETICAL_INDEX = YES -COLS_IN_ALPHA_INDEX = 1 -IGNORE_PREFIX = -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- -GENERATE_HTML = YES -HTML_OUTPUT = html -HTML_FILE_EXTENSION = .html -HTML_HEADER = -HTML_FOOTER = ./doc/doxygen/footer.html -HTML_STYLESHEET = -HTML_EXTRA_STYLESHEET = ./doc/doxygen/extra-stylesheet.css -HTML_ALIGN_MEMBERS = YES -GENERATE_HTMLHELP = NO -CHM_FILE = -HHC_LOCATION = -GENERATE_CHI = NO -BINARY_TOC = NO -TOC_EXPAND = NO -DISABLE_INDEX = NO -ENUM_VALUES_PER_LINE = 1 -GENERATE_TREEVIEW = NO -TREEVIEW_WIDTH = 250 -USE_MATHJAX = NO -MATHJAX_FORMAT = SVG -MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- -GENERATE_LATEX = YES -LATEX_OUTPUT = latex -LATEX_CMD_NAME = latex -MAKEINDEX_CMD_NAME = makeindex -COMPACT_LATEX = NO -PAPER_TYPE = letter -EXTRA_PACKAGES = amsmath amssymb mathrsfs -LATEX_HEADER = -PDF_HYPERLINKS = NO -USE_PDFLATEX = NO -LATEX_BATCHMODE = NO -LATEX_HIDE_INDICES = NO -FORMULA_FONTSIZE = 50 -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- -GENERATE_RTF = NO -RTF_OUTPUT = rtf -COMPACT_RTF = NO -RTF_HYPERLINKS = NO -RTF_STYLESHEET_FILE = -RTF_EXTENSIONS_FILE = -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- -GENERATE_MAN = YES -MAN_OUTPUT = man -MAN_EXTENSION = .3 -MAN_LINKS = NO -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- -GENERATE_XML = NO -XML_OUTPUT = xml -XML_SCHEMA = -XML_DTD = -XML_PROGRAMLISTING = YES -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- -GENERATE_AUTOGEN_DEF = NO -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- -GENERATE_PERLMOD = NO -PERLMOD_LATEX = NO -PERLMOD_PRETTY = YES -PERLMOD_MAKEVAR_PREFIX = -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = YES -EXPAND_ONLY_PREDEF = NO -SEARCH_INCLUDES = YES -INCLUDE_PATH = -INCLUDE_FILE_PATTERNS = -PREDEFINED = -EXPAND_AS_DEFINED = -SKIP_FUNCTION_MACROS = YES -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- -TAGFILES = -GENERATE_TAGFILE = -ALLEXTERNALS = NO -EXTERNAL_GROUPS = YES -PERL_PATH = /usr/bin/perl -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- -CLASS_DIAGRAMS = YES -HIDE_UNDOC_RELATIONS = YES -HAVE_DOT = YES -CLASS_GRAPH = YES -COLLABORATION_GRAPH = NO -GROUP_GRAPHS = YES -UML_LOOK = NO -TEMPLATE_RELATIONS = YES -INCLUDE_GRAPH = YES -INCLUDED_BY_GRAPH = YES -CALL_GRAPH = NO -CALLER_GRAPH = NO -GRAPHICAL_HIERARCHY = YES -DIRECTORY_GRAPH = YES -DOT_IMAGE_FORMAT = png -# Hack dark color support in through the dot path. Kind of cheating... -DOT_PATH = dot -Gbgcolor=black -DOTFILE_DIRS = -MAX_DOT_GRAPH_WIDTH = 800 -MAX_DOT_GRAPH_HEIGHT = 600 -MAX_DOT_GRAPH_DEPTH = 1000 -DOT_TRANSPARENT = NO -DOT_MULTI_TARGETS = NO -GENERATE_LEGEND = YES -DOT_CLEANUP = YES -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- -SEARCHENGINE = YES diff --git a/README.md b/README.md index f3472f52b3..2b21ee2c7d 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,27 @@ src="https://cdn.rawgit.com/mlpack/mlpack.org/e7d36ed8/mlpack-black.svg" style=" **mlpack** is an intuitive, fast, and flexible header-only C++ machine learning library with bindings to other languages. It is meant to be a machine learning analog to LAPACK, and aims to implement a wide array of machine learning methods -and functions as a "swiss army knife" for machine learning researchers. In -addition to its powerful C++ interface, mlpack also provides command-line +and functions as a "swiss army knife" for machine learning researchers. + +mlpack's lightweight C++ implementation makes it ideal for deployment, and it +can also be used for interactive prototyping via C++ notebooks (these can be +seen in action on mlpack's [homepage](https://www.mlpack.org/)). + +In addition to its powerful C++ interface, mlpack also provides command-line programs, Python bindings, Julia bindings, Go bindings and R bindings. +***Quick links:*** + + - Quickstart guides: [C++](doc/quickstart/cpp.md), + [CLI](doc/quickstart/cli.md), [Python](doc/quickstart/python.md), + [R](doc/quickstart/R.md), [Julia](doc/quickstart/julia.md), + [Go](doc/quickstart/go.md) + - [mlpack homepage](https://www.mlpack.org/) + - [mlpack documentation](https://www.mlpack.org/docs.html) + - [Examples repository](https://github.com/mlpack/examples/) + - [Tutorials](https://www.mlpack.org/doc/mlpack-git/doxygen/tutorials.html) + - [Development Site (Github)](https://www.github.com/mlpack/mlpack/) + [//]: # (numfocus-fiscal-sponsor-attribution) mlpack uses an [open governance model](./GOVERNANCE.md) and is fiscally @@ -50,32 +67,21 @@ variety of other needs.
-### 0. Contents +## 0. Contents and Quick Links - 1. [Introduction](#1-introduction) - 2. [Citation details](#2-citation-details) - 3. [Dependencies](#3-dependencies) - 4. [Building mlpack from source](#4-building-mlpack-from-source) - 5. [Running mlpack programs](#5-running-mlpack-programs) - 6. [Using mlpack from Python](#6-using-mlpack-from-python) - 7. [Further documentation](#7-further-documentation) - 8. [Bug reporting](#8-bug-reporting) - 9. [Building mlpack from vcpkg](#9-building-mlpack-from-vcpkg) + 1. [Citation details](#1-citation-details) + 2. [Dependencies](#2-dependencies) + 3. [Installing and using mlpack in C++](#4-installing-and-using-mlpack-in-c++) + 4. [Building mlpack bindings to other languages](#5-building-mlpack-bindings-to-other-languages) + 1. [Command-line programs](#4i-command-line-programs) + 2. [Python bindings](#4ii-python-bindings) + 3. [R bindings](#4iii-r-bindings) + 4. [Julia bindings](#4iv-julia-bindings) + 5. [Go bindings](#4v-go-bindings) + 5. [Building mlpack's test suite](#5-building-mlpacks-test-suite) + 6. [Further resources](#6-further-resources) -### 1. Introduction - -The mlpack website can be found at https://www.mlpack.org and it contains -numerous tutorials and extensive documentation. This README serves as a guide -for what mlpack is, how to install it, how to run it, and where to find more -documentation. The website should be consulted for further information: - - - [mlpack homepage](https://www.mlpack.org/) - - [mlpack documentation](https://www.mlpack.org/docs.html) - - [Tutorials](https://www.mlpack.org/doc/mlpack-git/doxygen/tutorials.html) - - [Development Site (Github)](https://www.github.com/mlpack/mlpack/) - - [API documentation (Doxygen)](https://www.mlpack.org/doc/mlpack-git/doxygen/index.html) - -### 2. Citation details +## 1. Citation details If you use mlpack in your research or software, please cite mlpack using the citation below (given in BibTeX format): @@ -96,266 +102,322 @@ citation below (given in BibTeX format): Citations are beneficial for the growth and improvement of mlpack. -### 3. Dependencies +## 2. Dependencies -mlpack has the following dependencies: +mlpack requires a C++14 compiler and has the following additional dependencies: - Armadillo >= 9.800 - CMake >= 3.6 - ensmallen >= 2.10.0 - cereal >= 1.1.2 - -All of those should be available in your distribution's package manager. If -not, you will have to compile each of them by hand. See the documentation for -each of those packages for more information. - -If you would like to use or build the mlpack Python bindings, make sure that the -following Python packages are installed: - - setuptools - cython >= 0.24 - numpy - pandas >= 0.15.0 - -If you would like to build the Julia bindings, make sure that Julia >= 1.3.0 is -installed. - -If you would like to build the Go bindings, make sure that Go >= 1.11.0 is -installed with this package: - - Gonum - -If you would like to build the R bindings, make sure that R >= 4.0 is -installed with these R packages. - - Rcpp >= 0.12.12 - RcppArmadillo >= 0.8.400.0 - RcppEnsmallen >= 0.2.10.0 - BH >= 1.58 - roxygen2 + - Armadillo >= 9.800 + - ensmallen >= 2.10.0 + - cereal >= 1.1.2 If the STB library headers are available, image loading support will be -compiled. +available. If you are compiling Armadillo by hand, ensure that LAPACK and BLAS are enabled. -### 4. Building mlpack from source +## 3. Installing and using mlpack in C++ -This document discusses how to build mlpack from source. These build directions -will work for any Linux-like shell environment (for example Ubuntu, macOS, -FreeBSD etc). However, mlpack is in the repositories of many Linux distributions -and so it may be easier to use the package manager for your system. For example, -on Ubuntu, you can install the mlpack library and command-line executables (e.g. -mlpack_pca, mlpack_kmeans etc.) with the following command: +*See also the [C++ quickstart](doc/quickstart/cpp.md).* - $ sudo apt-get install libmlpack-dev mlpack-bin +Since mlpack is a header-only library, installing just the headers for use in a +C++ application is trivial. From the root of the sources, configure and install +in the standard CMake way: -On Fedora or Red Hat (EPEL): - - $ sudo dnf install mlpack-devel mlpack-bin - -*Note*: Older Ubuntu versions may not have the most recent version of mlpack -available---for instance, at the time of this writing, Ubuntu 16.04 only has -mlpack 3.4.2 available. Options include upgrading your Ubuntu version, finding -a PPA or other non-official sources, or installing with a manual build. - -*Note*: If you are using RHEL7/CentOS 7, gcc 4.8 is too old to compile mlpack. -One option is to use `devtoolset-8`; see -[here](https://www.softwarecollections.org/en/scls/rhscl/devtoolset-8/) for more -information. - -There are some useful pages to consult in addition to this section: - - - [Building mlpack From Source](https://www.mlpack.org/doc/mlpack-git/doxygen/build.html) - - [Building mlpack From Source on Windows](https://www.mlpack.org/doc/mlpack-git/doxygen/build_windows.html) - -mlpack uses CMake as a build system and allows several flexible build -configuration options. You can consult any of the CMake tutorials for -further documentation, but this tutorial should be enough to get mlpack built -and installed. - -First, unpack the mlpack source and change into the unpacked directory. Here we -use mlpack-x.y.z where x.y.z is the version. - - $ tar -xzf mlpack-x.y.z.tar.gz - $ cd mlpack-x.y.z - -Then, make a build directory. The directory can have any name, but 'build' is -sufficient. - - $ mkdir build - $ cd build - -The next step is to run CMake to configure the project. Running CMake is the -equivalent to running `./configure` with autotools. If you run CMake with no -options, it will configure the project to build with no debugging symbols and -no profiling information: - - $ cmake ../ - -Options can be specified to compile with debugging information and profiling information: - - $ cmake -D DEBUG=ON -D PROFILE=ON ../ - -Options are specified with the -D flag. The allowed options include: - - DEBUG=(ON/OFF): compile with debugging symbols - PROFILE=(ON/OFF): compile with profiling symbols - ARMA_EXTRA_DEBUG=(ON/OFF): compile with extra Armadillo debugging symbols - ARMADILLO_INCLUDE_DIR=(/path/to/armadillo/include/): path to Armadillo headers - ARMADILLO_LIBRARY=(/path/to/armadillo/libarmadillo.so): Armadillo library - BUILD_CLI_EXECUTABLES=(ON/OFF): whether or not to build command-line programs - BUILD_PYTHON_BINDINGS=(ON/OFF): whether or not to build Python bindings - PYTHON_EXECUTABLE=(/path/to/python_version): Path to specific Python executable - PYTHON_INSTALL_PREFIX=(/path/to/python/): Path to root of Python installation - BUILD_JULIA_BINDINGS=(ON/OFF): whether or not to build Julia bindings - JULIA_EXECUTABLE=(/path/to/julia): Path to specific Julia executable - BUILD_GO_BINDINGS=(ON/OFF): whether or not to build Go bindings - GO_EXECUTABLE=(/path/to/go): Path to specific Go executable - BUILD_GO_SHLIB=(ON/OFF): whether or not to build shared libraries required by Go bindings - BUILD_R_BINDINGS=(ON/OFF): whether or not to build R bindings - R_EXECUTABLE=(/path/to/R): Path to specific R executable - BUILD_TESTS=(ON/OFF): whether or not to build tests - BUILD_SHARED_LIBS=(ON/OFF): compile shared libraries and executables as - opposed to static libraries - DISABLE_DOWNLOADS=(ON/OFF): whether to disable all downloads during build - ENSMALLEN_INCLUDE_DIR=(/path/to/ensmallen/include): path to include directory - for ensmallen - STB_IMAGE_INCLUDE_DIR=(/path/to/stb/include): path to include directory for - STB image library - USE_OPENMP=(ON/OFF): whether or not to use OpenMP if available - BUILD_DOCS=(ON/OFF): build Doxygen documentation, if Doxygen is available - (default ON) - -For example, to build mlpack's CLI bindings statically the following command can -be used: - - $ cmake -D BUILD_SHARED_LIBS=OFF ../ - -Other tools can also be used to configure CMake, but those are not documented -here. See [this section of the build guide](https://www.mlpack.org/doc/mlpack-git/doxygen/build.html#build_config) -for more details, including a full list of options, and their default values. - -By default, command-line programs will be built, and if the Python dependencies -(Cython, setuptools, numpy, pandas) are available, then Python bindings will -also be built. OpenMP will be used for parallelization when possible by -default. - -Once CMake is configured, building the library is as simple as typing 'make'. -This will build all library components and bindings. - - $ make - -If you do not want to build everything in the library, individual components -of the build can be specified: - - $ make mlpack_pca mlpack_knn mlpack_kfn - -If you want to build the tests, just make the `mlpack_test` target, and use -`ctest` to run the tests: - - $ make mlpack_test - $ ctest . - -If the build fails and you cannot figure out why, register an account on Github -and submit an issue. The mlpack developers will quickly help you figure it out: - -[mlpack on Github](https://www.github.com/mlpack/mlpack/) - -Alternately, mlpack help can be found in IRC at `#mlpack` on chat.freenode.net. - -If you wish to install mlpack to `/usr/local/include/mlpack/`, `/usr/local/lib/`, -and `/usr/local/bin/`, make sure you have root privileges (or write permissions -to those three directories), and simply type - - $ make install - -You can now run the executables by name; the mlpack headers are found in - `/usr/local/include/mlpack/` -and if Python bindings were built, you can access them with the `mlpack` -package in Python. - -### 5. Running mlpack programs - -After building mlpack, the executables will reside in `build/bin/`. You can -call them from there, or you can install the library and (depending on system -settings) they should be added to your PATH and you can call them directly. The -documentation below assumes the executables are in your PATH. - -Consider the 'mlpack_knn' program, which finds the k nearest neighbors in a -reference dataset of all the points in a query set. That is, we have a query -and a reference dataset. For each point in the query dataset, we wish to know -the k points in the reference dataset which are closest to the given query -point. - -Alternately, if the query and reference datasets are the same, the problem can -be stated more simply: for each point in the dataset, we wish to know the k -nearest points to that point. - -Each mlpack program has extensive help documentation which details what the -method does, what each of the parameters is, and how to use them: - -```shell -$ mlpack_knn --help +```sh +mkdir build && cd build/ +cmake ../ +sudo make install ``` -Running `mlpack_knn` on one dataset (that is, the query and reference -datasets are the same) and finding the 5 nearest neighbors is very simple: +You can add a few arguments to the `cmake` command to control the behavior of +the configuration and build process. Simply add these to the `cmake` command. +Some options are given below: -```shell -$ mlpack_knn -r dataset.csv -n neighbors_out.csv -d distances_out.csv -k 5 -v + - `-DCMAKE_INSTALL_PREFIX=/install/root/` will set the root of the install + directory to `/install/root` when `make install` is run. + - `-DDOWNLOAD_DEPENDENCIES=ON` will automatically download mlpack's + dependencies (ensmallen, Armadillo, and cereal). + - `-DDEBUG=ON` will enable debugging symbols in any compiled bindings or tests. + +There are also options to enable building bindings to each language that mlpack +supports; those are detailed in the following sections. + +Once headers are installed with `make install`, using mlpack in an application +consists only of including it. So, your program should include mlpack: + +```c++ +#include ``` -The `-v (--verbose)` flag is optional; it gives informational output. It is not -unique to `mlpack_knn` but is available in all mlpack programs. Verbose -output also gives timing output at the end of the program, which can be very -useful. +and when you link, be sure to link against Armadillo. If your example program +is `my_program.cpp`, your compiler is GCC, and you would like to compile with +OpenMP support (recommended) and optimizations, compile like this: -### 6. Using mlpack from Python - -If mlpack is installed to the system, then the mlpack Python bindings should be -automatically in your PYTHONPATH, and importing mlpack functionality into Python -should be very simple: - -```python ->>> from mlpack import knn +```sh +g++ -O3 -std=c++14 -o my_program my_program.cpp -larmadillo -fopenmp ``` -Accessing help is easy: +Note that if you want to serialize (save or load) neural networks, you should +add `#define MLPACK_ENABLE_ANN_SERIALIZATION` before including ``. -```python ->>> help(knn) +See the [C++ quickstart](doc/quickstart/cpp.md) and the +[examples](https://github.com/mlpack/examples) repository for some examples of +mlpack applications in C++, with corresponding `Makefile`s. + +### 3.1. Including mlpack and improving compile time + +mlpack is a template-heavy library, and if care is not used, compilation time of +a project can be increased greatly. Fortunately, there are a number of ways to +reduce compilation time: + + * Include individual headers, like ``, if you + are only using one component, instead of ``. This reduces the + amount of work the compiler has to do. + + * Only use the `MLPACK_ENABLE_ANN_SERIALIZATION` definition if you are + serializing neural networks in your code. When this define is enabled, + compilation time will increase significantly, as the compiler must generate + code for every possible type of layer. + + * If you are using mlpack in multiple .cpp files, consider using [`extern + templates`](https://isocpp.org/wiki/faq/cpp11-language-templates) so that the + compiler only instantiates each template once; add an explicit template + instantiation for each mlpack template type you want to use in a .cpp file, + and then use `extern` definitions elsewhere to let the compiler know it + exists in a different file. + +Other strategies exist too, such as precompiled headers, compiler options, +[`ccache`](https://ccache.dev), and others. + +## 4. Building mlpack bindings to other languages + +mlpack is not just a header-only library: it also comes with bindings to a +number of other languages, this allows flexible use of mlpack's efficient +implementations from languages that aren't C++. + +In general, you should *not* need to build these by hand---they should be +provided by either your system package manager or your language's package +manager. + +Building the bindings for a particular language is done by calling `cmake` with +different options; each example below shows how to configure an individual set +of bindings, but it is of course possible to combine the options and build +bindings for many languages at once. + +### 4.i. Command-line programs + +*See also the [command-line quickstart](doc/quickstart/cli.md).* + +The command-line programs have no extra dependencies. The set of programs that +will be compiled is detailed and documented on the [command-line program +documentation page](https://www.mlpack.org/doc/stable/cli_documentation.html). + +From the root of the mlpack sources, run the following commands to build and +install the command-line bindings: + +```sh +mkdir build && cd build/ +cmake -DBUILD_CLI_PROGRAMS=ON ../ +make +sudo make install ``` -The API is similar to the command-line programs. So, running `knn()` -(k-nearest-neighbor search) on the numpy matrix `dataset` and finding the 5 -nearest neighbors is very simple: +You can use `make -j`, where `N` is the number of cores on your machine, to +build in parallel; e.g., `make -j4` will use 4 cores to build. -```python ->>> output = knn(reference=dataset, k=5, verbose=True) +### 4.ii. Python bindings + +*See also the [Python quickstart](doc/quickstart/python.md).* + +mlpack's Python bindings are available on +[PyPI](https://pypi.org/project/mlpack) and +[conda-forge](https://conda-forge.org/packages/mlpack), and can be installed +with either `pip install mlpack` or `conda install -c conda-forge mlpack`. +These sources are recommended, as building the Python bindings by hand can be +complex. + +With that in mind, if you would still like to manually build the mlpack Python +bindings, first make sure that the following Python packages are installed: + + - setuptools + - cython >= 0.24 + - numpy + - pandas >= 0.15.0 + +Now, from the root of the mlpack sources, run the following commands to build +and install the Python bindings: + +```sh +mkdir build && cd build/ +cmake -DBUILD_PYTHON_BINDINGS=ON ../ +make +sudo make install ``` -This will store the output neighbors in `output['neighbors']` and the output -distances in `output['distances']`. Other mlpack bindings function similarly, -and the input/output parameters exactly match those of the command-line -programs. +You can use `make -j`, where `N` is the number of cores on your machine, to +build in parallel; e.g., `make -j4` will use 4 cores to build. You can also +specify a custom Python interpreter with the CMake option +`-DPYTHON_EXECUTABLE=/path/to/python`. -### 7. Further documentation +### 4.iii. R bindings -The documentation given here is only a fraction of the available documentation -for mlpack. If doxygen is installed, you can type `make doc` to build the -documentation locally. Alternately, up-to-date documentation is available for -older versions of mlpack: +*See also the [R quickstart](doc/quickstart/R.md).* - - [mlpack homepage](https://www.mlpack.org/) - - [mlpack documentation](https://www.mlpack.org/docs.html) - - [Tutorials](https://www.mlpack.org/doc/mlpack-git/doxygen/tutorials.html) - - [Development Site (Github)](https://www.github.com/mlpack/mlpack/) - - [API documentation (Doxygen)](https://www.mlpack.org/doc/mlpack-git/doxygen/index.html) +mlpack's R bindings are available as the R package +[mlpack](https://cran.r-project.org/web/packages/mlpack/index.html) on CRAN. +You can install the package by running `install.packages('mlpack')`, and this is +the recommended way of getting mlpack in R. + +If you still wish to build the R bindings by hand, first make sure the following +dependencies are installed: + + - R >= 4.0 + - Rcpp >= 0.12.12 + - RcppArmadillo >= 0.9.800.0 + - RcppEnsmallen >= 0.2.10.0 + - roxygen2 + - testthat + - pkgbuild + +These can be installed with `install.packages()` inside of your R environment. +Once the dependencies are available, you can configure mlpack and build the R +bindings by running the following commands from the root of the mlpack sources: + +```sh +mkdir build && cd build/ +cmake -DBUILD_R_BINDINGS=ON ../ +make +sudo make install +``` + +You may need to specify the location of the R program in the `cmake` command +with the option `-DR_EXECUTABLE=/path/to/R`. + +Once the build is complete, a tarball can be found under the build directory in +`src/mlpack/bindings/R/`, and then that can be installed into your R environment +with a command like `install.packages(mlpack_3.4.3.tar.gz, repos=NULL, +type='source')`. + +### 4.iv. Julia bindings + +*See also the [Julia quickstart](doc/quickstart/julia.md).* + +mlpack's Julia bindings are available by installing the +[mlpack.jl](https://github.com/mlpack/mlpack.jl) package using +`Pkg.add("mlpack.jl")`. The process of building, packaging, and distributing +mlpack's Julia bindings is very nontrivial, so it is recommended to simply use +the version available in `Pkg`, but if you want to build the bindings by hand +anyway, you can configure and build them by running the following commands from +the root of the mlpack sources: + +```sh +mkdir build && cd build/ +cmake -DBUILD_JULIA_BINDINGS=ON ../ +make +``` + +If CMake cannot find your Julia installation, you can add +`-DJULIA_EXECUTABLE=/path/to/julia` to the CMake configuration step. + +Note that the `make install` step is not done above, since the Julia binding +build system was not meant to be installed directly. Instead, to use handbuilt +bindings (for instance, to test them), one option is to start Julia with +`JULIA_PROJECT` set as an environment variable: + +```sh +cd build/src/mlpack/bindings/julia/mlpack/ +JULIA_PROJECT=$PWD julia +``` + +and then `using mlpack` should work. + +### 4.v. Go bindings + +*See also the [Go quickstart](doc/quickstart/go.md).* + +To build mlpack's Go bindings, ensure that Go >= 1.11.0 is installed, and that +the Gonum package is available. You can use `go get` to install mlpack for Go: + +```sh +go get -u -d mlpack.org/v1/mlpack +cd ${GOPATH}/src/mlpack.org/v1/mlpack +make install +``` + +The process of building the Go bindings by hand is a little tedious, so +following the steps above is recommended. However, if you wish to build the Go +bindings by hand anyway, you can do this by running the following commands from +the root of the mlpack sources: + +```sh +mkdir build && cd build/ +cmake -DBUILD_GO_BINDINGS=ON ../ +make +sudo make install +``` + +## 5. Building mlpack's test suite + +mlpack contains an extensive test suite that exercises every part of the +codebase. It is easy to build and run the tests with CMake and CTest, as below: + +```sh +mkdir build && cd build/ +cmake -DBUILD_TESTS=ON ../ +make +ctest . +``` + +If you want to test the bindings, too, you will have to adapt the CMake +configuration command to turn on the language bindings that you want to +test---see the previous sections for details. + +## 6. Further Resources + +More documentation is available for both users and developers. + +***User documentation***: + + - [File formats and loading data in mlpack](doc/user/formats.md) + - [Matrices in mlpack](doc/user/matrices.md) + - [Cross-Validation](doc/user/cv.md) + - [Hyper-parameter Tuning](doc/user/hpt.md) + - [Building mlpack from source on Windows](doc/user/build_windows.md) + - [Sample C++ ML App for Windows](doc/user/sample_ml_app.md) + - [Examples repository](https://github.com/mlpack/examples/) + +***Tutorials:*** + + - [Alternating Matrix Factorization (AMF)](doc/tutorials/amf.md) + - [Artificial Neural Networks (ANN)](doc/tutorials/ann.md) + - [Approximate k-Furthest Neighbor Search (`approx_kfn`)](doc/tutorials/approx_kfn.md) + - [Collaborative Filtering (CF)](doc/tutorials/cf.md) + - [DatasetMapper](doc/tutorials/datasetmapper.md) + - [Density Estimation Trees (DET)](doc/tutorials/det.md) + - [Euclidean Minimum Spanning Trees (EMST)](doc/tutorials/emst.md) + - [Fast Max-Kernel Search (FastMKS)](doc/tutorials/fastmks.md) + - [Image Utilities](doc/tutorials/image.md) + - [k-Means Clustering](doc/tutorials/kmeans.md) + - [Linear Regression](doc/tutorials/linear_regression.md) + - [Neighbor Search (k-Nearest-Neighbors)](doc/tutorials/neighbor_search.md) + - [Range Search](doc/tutorials/range_search.md) + - [Reinforcement Learning](doc/tutorials/reinforcement_learning.md) + +***Developer documentation***: + + - [mlpack versions in code](doc/developer/version.md) + - [Writing an mlpack binding](doc/developer/iodoc.md) + - [mlpack Timers](doc/developer/timer.md) + - [mlpack automatic bindings to other languages](doc/developer/bindings.md) + - [The ElemType policy in mlpack](doc/developer/elemtype.md) + - [The KernelType policy in mlpack](doc/developer/kernels.md) + - [The MetricType policy in mlpack](doc/developer/metrics.md) + - [The TreeType policy in mlpack](doc/developer/trees.md) To learn about the development goals of mlpack in the short- and medium-term future, see the [vision document](https://www.mlpack.org/papers/vision.pdf). +<<<<<<< HEAD ### 8. Bug reporting (see also [mlpack help](https://www.mlpack.org/questions.html)) @@ -379,16 +441,8 @@ and the git commit list is available at Lastly, the IRC channel `#mlpack` on Freenode can be used to get help. -### 9. Building mlpack from vcpkg - -The mlpack port in vcpkg is kept up to date by Microsoft team members and community contributors. The url of vcpkg is: https://github.com/Microsoft/vcpkg . You can download and install mlpack using the vcpkg dependency manager: - -```shell -git clone https://github.com/Microsoft/vcpkg.git -cd vcpkg -./bootstrap-vcpkg.sh # ./bootstrap-vcpkg.bat for Windows -./vcpkg integrate install -./vcpkg install mlpack -``` - -If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. +If you have problems, find a bug, or need help, you can try visiting +the [mlpack help](https://www.mlpack.org/questions.html) page, or [mlpack on +Github](https://www.github.com/mlpack/mlpack/). Alternately, mlpack help can be +found on Matrix at `#mlpack`; see also the +[community](https://www.mlpack.org/community.html) page. diff --git a/doc/guide/bindings.hpp b/doc/developer/bindings.md similarity index 56% rename from doc/guide/bindings.hpp rename to doc/developer/bindings.md index b6b93e67c7..9b3d2e92b2 100644 --- a/doc/guide/bindings.hpp +++ b/doc/developer/bindings.md @@ -1,6 +1,4 @@ -/*! @page bindings mlpack automatic bindings to other languages - -@section bindings_overview Overview +# mlpack automatic bindings to other languages mlpack has a system to automatically generate bindings to other languages, such as Python and command-line programs, and it is extensible to other languages @@ -16,27 +14,15 @@ curious enough to see how the sausage is made. The document is split into several sections: - - @ref bindings_intro - - @ref bindings_code - - @ref bindings_general - - @ref bindings_general_program_doc - - @ref bindings_general_define_params - - @ref bindings_general_functions - - @ref bindings_general_more - - @ref bindings_structure - - @ref bindings_cli - - @ref bindings_cli_mlpack_main - - @ref bindings_cli_matrix - - @ref bindings_cli_parsing - - @ref bindings_python - - @ref bindings_python_matrix - - @ref bindings_python_model - - @ref bindings_python_setup_py - - @ref bindings_python_build_pyx - - @ref bindings_python_testing - - @ref bindings_new + - [Introduction](#introduction) + - [Writing code that can be turned into a binding](#writing-code-that-can-be-turned-into-a-binding) + - [How to write mlpack bindings](#how-to-write-mlpack-bindings) + - [Structure of IO module and associated macros](#structure-of-io-module-and-associated-macros) + - [Command-line program bindings](#command-line-program-bindings) + - [Python bindings](#python-bindings) + - [Adding new binding types](#adding-new-binding-types) -@section bindings_intro Introduction +## Introduction C++ is not the most popular language on the planet, and it (unfortunately) can scare many away with its ultra-verbose error messages, confusing template rules, @@ -44,10 +30,10 @@ and complex metaprogramming techniques. Most practitioners of machine learning tend to avoid writing native C++ and instead prefer other languages---probably most notably Python. -In the case of Python, many projects will use tools like SWIG -(http://www.swig.org/) to automatically generate bindings, or they might +In the case of Python, many projects will use tools like +[SWIG](http://www.swig.org) to automatically generate bindings, or they might hand-write Cython. The same types of strategies may be used for other -languages; hand-written MEX files may be used for MATLAB, hand-written RCpp +languages; hand-written MEX files may be used for MATLAB, hand-written Rcpp bindings might be used for R bindings, and so forth. However, these approaches have a fundamental flaw: the hand-written bindings @@ -59,7 +45,7 @@ workload; therefore an alternate solution is needed. At the time of the design of this system, mlpack shipped headers for a C++ library as well as many (~40) hand-written command-line programs that used the -mlpack::IO object to manage command-line arguments. These programs all had +`mlpack::IO` object to manage command-line arguments. These programs all had similar structure, and could be logically split into three sections: - parse the input options supplied by the user @@ -68,21 +54,21 @@ similar structure, and could be logically split into three sections: The user might interface with this command-line program like the following: -@code +```sh $ mlpack_knn -r reference.csv -q query.csv -k 3 -d d.csv -n n.csv -@endcode +``` That is, they would pass a number of input options---some were numeric values -(like @c -k @c 3 ); some were filenames (like @c -r @c reference.csv ); and a -few other types also. Therefore, the first stage of the program---parsing input +(like `-k 3`); some were filenames (like `-r reference.csv`); and a few other +types also. Therefore, the first stage of the program---parsing input options---would be handled by reading the command line and loading any input matrices. Preparing the output, which usually consists of data matrices (i.e. -@c -d @c d.csv ) involves saving the matrix returned by the algorithm to the -user's desired file. +`-d d.csv`) involves saving the matrix returned by the algorithm to the user's +desired file. Ideally, any binding to any language would have this same structure, and the actual "run the machine learning algorithm" code could be identical. For -MATLAB, for instance, we would not need to read the file @c reference.csv but +MATLAB, for instance, we would not need to read the file `reference.csv` but instead the user would simply pass their data matrix as an argument. So each input and output parameter would need to be handled differently, but the algorithm could be run identically across all bindings. @@ -91,33 +77,30 @@ Therefore, design of an automatically-generated binding system would simply involve generating the boilerplate code necessary to parse input options for a given language, and to return output options to a user. -@section bindings_code Writing code that can be turned into a binding +## Writing code that can be turned into a binding This section details what a binding file might actually look like. It is good to have this API in mind when reading the following sections. -Each mlpack binding is typically contained in the @c src/mlpack/methods/ folder +Each mlpack binding is typically contained in the `src/mlpack/methods/` folder corresponding to a given machine learning algorithm, with the suffix -@c _main.cpp ; so an example is @c src/mlpack/methods/pca/pca_main.cpp . +`_main.cpp`; so an example is `src/mlpack/methods/pca/pca_main.cpp`. These files have roughly two parts: - - definition of the input and output parameters with @c PARAM macros and - documentation with @c BINDING macros - - implementation of @c BINDING_FUNCTION(), which is the actual machine learning + - definition of the input and output parameters with `PARAM` macros and + documentation with `BINDING` macros + - implementation of `BINDING_FUNCTION()`, which is the actual machine learning code Here is a simple example file: -@code +```c++ // This is a stripped version of mean_shift_main.cpp. -#include -#include +#include // Define the name of the binding (as seen by the binding generation system). -#ifdef BINDING_NAME - #undef BINDING_NAME -#endif +#undef BINDING_NAME #define BINDING_NAME mean_shift #include @@ -244,104 +227,98 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) if (params.Has("centroid")) params.Get("centroid") = std::move(centroids); } -@endcode +``` -We can see that we have defined the name of the binding with the @c BINDING_NAME -macro, and basic program information in the @c BINDING_USER_NAME(), @c -BINDING_SHORT_DESC(), @c BINDING_LONG_DESC(), @c BINDING_EXAMPLE() and @c -BINDING_SEE_ALSO() macros. This is, for instance, what is displayed to describe -the binding if the user passed the \--help option for a command-line +We can see that we have defined the name of the binding with the `BINDING_NAME` +macro, and basic program information in the `BINDING_USER_NAME()`, +`BINDING_SHORT_DESC()`, `BINDING_LONG_DESC()`, `BINDING_EXAMPLE()` and +`BINDING_SEE_ALSO()` macros. This is, for instance, what is displayed to +describe the binding if the user passed the `--help` option for a command-line program. Then, we define five parameters, three input and two output, that define the data and options that the mean shift clustering will function on. These -parameters are defined with the @c PARAM macros, of which there are many. The +parameters are defined with the `PARAM` macros, of which there are many. The names of these macros specify the type, whether the parameter is required, and whether the parameter is input or output. Some examples: - - @c PARAM_STRING_IN() -- a string-type input parameter - - @c PARAM_MATRIX_OUT() -- a matrix-type output parameter - - @c PARAM_DOUBLE_IN_REQ() -- a required double-type input parameter - - @c PARAM_UMATRIX_IN() -- an unsigned matrix-type input parameter - - @c PARAM_MODEL_IN() -- a serializable model-type input parameter + - `PARAM_STRING_IN()` -- a string-type input parameter + - `PARAM_MATRIX_OUT()` -- a matrix-type output parameter + - `PARAM_DOUBLE_IN_REQ()` -- a required double-type input parameter + - `PARAM_UMATRIX_IN()` -- an unsigned matrix-type input parameter + - `PARAM_MODEL_IN()` -- a serializable model-type input parameter Note that each of these macros may have slightly different syntax. See the links above for further documentation. -In order to write a new binding, then, you simply must define @c BINDING_NAME, -then write @c BINDING_USER_NAME(), @c BINDING_SHORT_DESC(), @c -BINDING_LONG_DESC(), @c BINDING_EXAMPLE() and @c BINDING_SEE_ALSO() definitions -of the program with some docuentation, define the input and output parameters as -@c PARAM macros, and then write a @c BINDING_FUNCTION() function that actually -performs the functionality of the binding. +In order to write a new binding, then, you simply must define `BINDING_NAME`, +then write `BINDING_USER_NAME()`, `BINDING_SHORT_DESC()`, `BINDING_LONG_DESC()`, +`BINDING_EXAMPLE()` and `BINDING_SEE_ALSO()` definitions of the program with +some docuentation, define the input and output parameters as `PARAM` macros, and +then write a `BINDING_FUNCTION()` function that actually performs the +functionality of the binding. -Inside of @c BINDING_FUNCTION(util::Params& params, util::Timers& timers): +Inside of `BINDING_FUNCTION(util::Params& params, util::Timers& timers)`: - - All input parameters are accessible through @c params.Get("name"). + - All input parameters are accessible through `params.Get("name")`. - All output parameters should be set by the end of the function with the - @c params.Get("name") method. - - The @c params.Has("name") function will return @c true if the parameter - @c "name" was specified. - - Timers can be started and stopped with @c timers.Start("timer_name") and - @c timers.Stop("timer_name"). + `params.Get("name")` method. + - The `params.Has("name")` function will return `true` if the parameter + `"name"` was specified. + - Timers can be started and stopped with `timers.Start("timer_name")` and + `timers.Stop("timer_name")`. -Then, assuming that your program is saved in the file @c program_name_main.cpp, +Then, assuming that your program is saved in the file `program_name_main.cpp`, generating bindings for other languages is a simple addition to the -@c CMakeLists.txt file: +`CMakeLists.txt` file in `src/mlpack/methods/CMakeLists.txt`: -@code -add_cli_executable(program_name) -add_python_binding(program_name) -add_markdown_docs(program_name "cli;python" "category") -@endcode +``` +add_all_bindings(program_dir program_name "category") +``` -In this example, @c add_markdown_docs() will generate documentation that is -typically used to build the website. The "category" parameter should be one of -the categories in @c src/mlpack/bindings/markdown/MarkdownCategories.cmake. +In this example, this will also add a Markdown binding, which will generate +documentation that is typically used to build the website. The `category` +parameter should be one of the categories in +`src/mlpack/bindings/markdown/MarkdownCategories.cmake`. -@section bindings_general How to write mlpack bindings +## How to write mlpack bindings This section describes the general structure of the automatic binding system and how one might write a new binding for mlpack. After reading this section it should be relatively clear how one could use the provided functionality in the -@c Params and @c Timers class along with CMake to add a binding for a new mlpack +`Params` and `Timers` class along with CMake to add a binding for a new mlpack machine learning method. If it is not clear, then the examples in the following sections should clarify. -@subsection bindings_general_binding_name Providing a name with @c BINDING_NAME +### Providing a name with `BINDING_NAME` -Every binding must have the macro @c BINDING_NAME defined, specifying a name +Every binding must have the macro `BINDING_NAME` defined, specifying a name (without spaces, generally all lowercase) that will be used to represent the -binding. It is suggested to @c #undef any previous setting of @c BINDING_NAME +binding. It is suggested to `#undef` any previous setting of `BINDING_NAME` just to prevent any strange error messages in case it is already defined. Here is an example that can be adapted: -@code -#ifdef BINDING_NAME - #undef BINDING_NAME -#endif +```c++ +#undef BINDING_NAME #define BINDING_NAME my_binding_name // BINDING_NAME should be defined before including mlpack_main.hpp! #include -@endcode +``` If this macro is not defined, compilation of the binding will fail in many ways with potentially obscure error messages! (Sorry that they are bad error messages. The preprocessor doesn't give us too much to work with.) -@subsection bindings_general_program_doc Documenting a program with -@c BINDING_USER_NAME(), @c BINDING_SHORT_DESC(), @c BINDING_LONG_DESC(), -@c BINDING_EXAMPLE() and @c BINDING_SEE_ALSO(). +### Documenting a program with macros -Any mlpack program should be documented with the @c BINDING_USER_NAME(), -@c BINDING_SHORT_DESC(), @c BINDING_LONG_DESC() , @c BINDING_EXAMPLE() and -@c BINDING_SEE_ALSO() macros, which is available from the -@c header. The macros -are of the form +Any mlpack binding should be documented with the `BINDING_USER_NAME()`, +`BINDING_SHORT_DESC()`, `BINDING_LONG_DESC()`, `BINDING_EXAMPLE()` and +`BINDING_SEE_ALSO()` macros, which is available from the +`` header. The macros are of the form -@code +```c++ BINDING_USER_NAME("program name"); BINDING_SHORT_DESC("This is a short, two-sentence description of what the program does."); BINDING_LONG_DESC("This is a long description of what the program does." @@ -352,17 +329,17 @@ BINDING_EXAMPLE("This contains another example for this particular binding.\n" + PROGRAM_CALL(...)); // There could be many of these "see alsos". BINDING_SEE_ALSO("https://en.wikipedia.org/wiki/Machine_learning"); -@endcode +``` The short documentation should be two sentences indicating what the program implements and does, and a quick overview of how it can be used and what it should be used for. When writing new short documentation, it is a good idea to take a look at the existing documentation to get an idea of the general format. -For the "see also" section, you can specify as many @c SEE_ALSO() calls as you +For the "see also" section, you can specify as many `SEE_ALSO()` calls as you see fit. These are links used at the "see also" section of the website documentation for each binding, and it's very important that relevant links are -provided (also to other bindings). See the @c SEE_ALSO() documentation for more +provided (also to other bindings). See the `SEE_ALSO()` documentation for more details. Although it is possible to provide very short documentation, it is certainly @@ -377,7 +354,7 @@ immediately search for, instead of taking a long time to read and carefully consider all of the written documentation. However, it is difficult to write language-agnostic documentation. For -instance, in a command-line program, an output parameter '\--output_file' would +instance, in a command-line program, an output parameter `--output_file` would be specified on the command line as an input parameter, but in Python, the output parameter 'output' would actually simply be returned from the call to the Python function. Therefore, we must be careful how our documentation refers to @@ -388,18 +365,18 @@ input and output parameters. The following general guidelines can help: like Python and MATLAB and also "arguments given on the command line" for command line programs. - - Use the provided @c PRINT_PARAM_STRING() macro to print the names of - parameters. For instance, PRINT_PARAM_STRING("shuffle") will print - @c '\--shuffle' for a command line program and @c 'shuffle' for a Python - binding. The @c PRINT_PARAM_STRING() macro also takes into account the type + - Use the provided `PRINT_PARAM_STRING()` macro to print the names of + parameters. For instance, `PRINT_PARAM_STRING("shuffle")` will print + `--shuffle` for a command line program and `'shuffle'` for a Python + binding. The `PRINT_PARAM_STRING()` macro also takes into account the type of the parameter. - - Use the provided @c PRINT_DATASET() and @c PRINT_MODEL() macro to introduce + - Use the provided `PRINT_DATASET()` and `PRINT_MODEL()` macro to introduce example datasets or models, which can be useful when introducing an example - usage of the program. So you could write @c '"to @c run @c with @c a - @c dataset @c " @c + @c PRINT_DATASET("data") @c + @c "..."'. + usage of the program. So you could write `"to run with a dataset " + + PRINT_DATASET("data") + "..."`. - - Use the provided @c PRINT_CALL() macro to print example invocations of the + - Use the provided `PRINT_CALL()` macro to print example invocations of the program. The first argument is the name of the program, and then the following arguments should be the name of a parameter followed by the value of that parameter. @@ -410,95 +387,123 @@ input and output parameters. The following general guidelines can help: - Remember that some languages give output through return values and some give output using other input parameters. So the right verbiage to use is, e.g., - 'the results may be saved using the PRINT_PARAM_STRING("output") - parameter', and @b not 'the results are returned through the - PRINT_PARAM_STRING("output") parameter'. + `the results may be saved using the PRINT_PARAM_STRING("output") parameter`, + and ***not*** `the results are returned through the + PRINT_PARAM_STRING("output") parameter`. -Each of these macros (@c PRINT_PARAM_STRING(), @c PRINT_DATASET(), -@c PRINT_MODEL(), and @c PRINT_CALL() ) provides different output depending on -the language. Below are some example of documentation strings and their outputs -for different languages. Note that the output might not be *exactly* as written -or formatted here, but the general gist should be the same. +Each of these macros (`PRINT_PARAM_STRING()`, `PRINT_DATASET()`, +`PRINT_MODEL()`, and `PRINT_CALL()`) provides different output depending on the +language. Below are some example of documentation strings and their outputs for +different languages. Note that the output might not be *exactly* as written or +formatted here, but the general gist should be the same. -@code -Input C++ (snippet): +*Input C++ (snippet):* +```c++ "The parameter " + PRINT_PARAM_STRING("shuffle") + ", if set, will shuffle " "the data before learning." +``` -Command-line program output (snippet): +*Command-line program output (snippet):* +``` The parameter '--shuffle', if set, will shuffle the data before learning. +``` -Python binding output (snippet): +*Python binding output (snippet):* +``` The parameter 'shuffle', if set, will shuffle the data before learning. +``` -Julia binding output (snippet): +*Julia binding output (snippet):* +``` The parameter `shuffle`, if set, will shuffle the data before learning. +``` -Go binding output (snippet): +*Go binding output (snippet):* +``` The parameter "Shuffle", if set, will shuffle the data before learning. -@endcode +``` -@code -Input C++ (snippet): +Another example: +*Input C++ (snippet):* + +```c++ "The output matrix can be saved with the " + PRINT_PARAM_STRING("output") + " output parameter." +``` -Command-line program output (snippet): +*Command-line program output (snippet):* +``` The output matrix can be saved with the '--output_file' output parameter. +``` -Python binding output (snippet): +*Python binding output (snippet):* +``` The output matrix can be saved with the 'output' output parameter. +``` -Julia binding output (snippet): +*Julia binding output (snippet):* +``` The output matrix can be saved with the `output` output parameter. +``` -Go binding output (snippet): +*Go binding output (snippet):* +``` The output matrix can be saved with the "output" output parameter. -@endcode +``` -@code -Input C++ (snippet): +And another example: +*Input C++ (snippet):* + +```c++ "For example, to train a model on the dataset " + PRINT_DATASET("x") + " and " "save the output model to " + PRINT_MODEL("model") + ", the following command" " can be used:" "\n\n" + PRINT_CALL("program", "input", "x", "output_model", "model") +``` -Command-line program output (snippet): +*Command-line program output (snippet):* +``` For example, to train a model on the dataset 'x.csv' and save the output model to 'model.bin', the following command can be used: $ program --input_file x.csv --output_model_file model.bin +``` -Python binding output (snippet): +*Python binding output (snippet):* +``` For example, to train a model on the dataset 'x' and save the output model to 'model', the following command can be used: >>> output = program(input=x) >>> model = output['output_model'] +``` -Julia binding output (snippet): +*Julia binding output (snippet):* +``` For example, to train a model on the dataset `x` and save the output model to `model`, the following command can be used: julia> model = program(input=x) +``` -Go binding output (snippet): +*Go binding output (snippet):* +``` For example, to train a model on the dataset "x" and save the output model to "model", the following command can be used: @@ -507,11 +512,13 @@ Go binding output (snippet): param.Input = x model := mlpack.Program(param) -@endcode +``` -@code -Input C++ (full program, 'random_numbers_main.cpp'): +And finally, a full program example: +*Input C++ (full program, `random_numbers_main.cpp`):* + +```c++ // Program Name. BINDING_USER_NAME("Random Numbers"); @@ -543,11 +550,11 @@ Input C++ (full program, 'random_numbers_main.cpp'): "\n\n" + PRINT_CALL("random_numbers", "num_values", 100, "subtract", 3, "output", "rand", "output_model", "rand_lr")); -@endcode +``` -Command line output: +*Command line output*: -@code +``` Random Numbers This program generates random numbers with a variety of nonsensical @@ -567,11 +574,11 @@ Command line output: $ random_numbers --num_values 100 --subtract 3 --output_file rand.csv --output_model_file rand_lr.bin -@endcode +``` -Python binding output: +*Python binding output*: -@code +``` Random Numbers This program generates random numbers with a variety of nonsensical @@ -592,11 +599,11 @@ Python binding output: >>> output = random_numbers(num_values=100, subtract=3) >>> rand = output['output'] >>> rand_lr = output['output_model'] -@endcode +``` -Julia binding output: +*Julia binding output:* -@code +``` Random Numbers This program generates random numbers with a variety of nonsensical @@ -617,11 +624,11 @@ Julia binding output: ```julia julia> rand, rand_lr = random_numbers(num_values=100, subtract=3) ``` -@endcode +``` -Go binding output: +*Go binding output:* -@code +``` Random Numbers This program generates random numbers with a variety of nonsensical @@ -645,128 +652,121 @@ Go binding output: param.Subtract=3 rand, randLr := mlpack.RandomNumbers(param) -@endcode +``` -@subsection bindings_general_define_params Defining parameters for a program +### Defining parameters for a program -There exist several macros that can be used after a @c BINDING_LONG_DESC() and -@c BINDING_EXAMPLE() definition to define the parameters that can be specified +There exist several macros that can be used after a `BINDING_LONG_DESC()` and +`BINDING_EXAMPLE()` definition to define the parameters that can be specified for a given mlpack program. These macros all have the same general definition: the name of the macro specifies the type of the parameter, whether or not the -parameter is required, and whether the parameter is an input or output parameter. -Then as arguments to the macros, the name, description, and sometimes the -single-character alias and the default value of the parameter. +parameter is required, and whether the parameter is an input or output +parameter. Then as arguments to the macros, the name, description, and +sometimes the single-character alias and the default value of the parameter. To give a flavor of how these definitions look, the definition -@code +```c++ PARAM_STRING_IN("algorithm", "The algorithm to use: 'svd' or 'blah'.", "a"); -@endcode +``` -will define a string input parameter @c algorithm (referenced as -@c '\--algorithm' from the command-line or @c 'algorithm' from Python) with the -description The algorithm to use: 'svd' or 'blah'. The -single-character alias @c '-a' can be used from a command-line program (but -means nothing in Python). +will define a string input parameter `algorithm` (referenced as `--algorithm` +from the command-line or `'algorithm'` from Python) with the description `The +algorithm to use: 'svd' or 'blah'.` The single-character alias `-a` can be used +from a command-line program (but means nothing in Python). There are numerous different macros that can be used: - - @c PARAM_FLAG() - boolean flag parameter - - @c PARAM_INT_IN() - integer input parameter - - @c PARAM_INT_OUT() - integer output parameter - - @c PARAM_DOUBLE_IN() - double input parameter - - @c PARAM_DOUBLE_OUT() - double output parameter - - @c PARAM_STRING_IN() - string input parameter - - @c PARAM_STRING_OUT() - string output parameter - - @c PARAM_MATRIX_IN() - double-valued matrix (arma::mat) input + - `PARAM_FLAG()` - boolean flag parameter + - `PARAM_INT_IN()` - integer input parameter + - `PARAM_INT_OUT()` - integer output parameter + - `PARAM_DOUBLE_IN()` - double input parameter + - `PARAM_DOUBLE_OUT()` - double output parameter + - `PARAM_STRING_IN()` - string input parameter + - `PARAM_STRING_OUT()` - string output parameter + - `PARAM_MATRIX_IN()` - double-valued matrix (`arma::mat`) input parameter + - `PARAM_MATRIX_OUT()` - double-valued matrix (`arma::mat`) output parameter + - `PARAM_UMATRIX_IN()` - size_t-valued matrix (`arma::Mat`) input parameter - - @c PARAM_MATRIX_OUT() - double-valued matrix (arma::mat) output + - `PARAM_UMATRIX_OUT()` - size_t-valued matrix (`arma::Mat`) output parameter - - @c PARAM_UMATRIX_IN() - size_t-valued matrix (arma::Mat) - input parameter - - @c PARAM_UMATRIX_OUT() - size_t-valued matrix (arma::Mat) - output parameter - - @c PARAM_TMATRIX_IN() - transposed double-valued matrix (arma::mat) - input parameter - - @c PARAM_TMATRIX_OUT() - transposed double-valued matrix (arma::mat) - output parameter - - @c PARAM_MATRIX_AND_INFO_IN() - matrix with categoricals input parameter - (std::tuple) - - @c PARAM_COL_IN() - double-valued column vector (arma::vec) input + - `PARAM_TMATRIX_IN()` - transposed double-valued matrix (`arma::mat`) input parameter - - @c PARAM_COL_OUT() - double-valued column vector (arma::vec) output + - `PARAM_TMATRIX_OUT()` - transposed double-valued matrix (`arma::mat`) output parameter - - @c PARAM_UCOL_IN() - size_t-valued column vector (arma::Col) - input parameter - - @c PARAM_UCOL_OUT() - size_t-valued column vector - (arma::Col) output parameter - - @c PARAM_ROW_IN() - double-valued row vector (arma::rowvec) input + - `PARAM_MATRIX_AND_INFO_IN()` - matrix with categoricals input parameter + (`std::tuplearma::rowvec) output + - `PARAM_UCOL_IN()` - size_t-valued column vector (`arma::Col`) input parameter - - @c PARAM_VECTOR_IN() - std::vector input parameter - - @c PARAM_VECTOR_OUT() - std::vector output parameter - - @c PARAM_MODEL_IN() - serializable model input parameter - - @c PARAM_MODEL_OUT() - serializable model output parameter + - `PARAM_UCOL_OUT()` - size_t-valued column vector (`arma::Col`) output + parameter + - `PARAM_ROW_IN()` - double-valued row vector (`arma::rowvec`) input parameter + - `PARAM_ROW_OUT()` - double-valued row vector (`arma::rowvec`) output + parameter + - `PARAM_VECTOR_IN()` - `std::vector` input parameter + - `PARAM_VECTOR_OUT()` - `std::vector` output parameter + - `PARAM_MODEL_IN()` - serializable model input parameter + - `PARAM_MODEL_OUT()` - serializable model output parameter And for input parameters, the parameter may also be required: - - @c PARAM_INT_IN_REQ() - - @c PARAM_DOUBLE_IN_REQ() - - @c PARAM_STRING_IN_REQ() - - @c PARAM_MATRIX_IN_REQ() - - @c PARAM_UMATRIX_IN_REQ() - - @c PARAM_TMATRIX_IN_REQ() - - @c PARAM_VECTOR_IN_REQ() - - @c PARAM_MODEL_IN_REQ() + - `PARAM_INT_IN_REQ()` + - `PARAM_DOUBLE_IN_REQ()` + - `PARAM_STRING_IN_REQ()` + - `PARAM_MATRIX_IN_REQ()` + - `PARAM_UMATRIX_IN_REQ()` + - `PARAM_TMATRIX_IN_REQ()` + - `PARAM_VECTOR_IN_REQ()` + - `PARAM_MODEL_IN_REQ()` -Click the links for each macro to read further documentation. Note also that -each possible combination of @c IN, @c OUT, and @c REQ is not available---output -options cannot be required, and some combinations simply have not been added -because they have not been needed. +See the source documentation for each macro to read further details. Note also +that each possible combination of `IN`, `OUT`, and `REQ` is not +available---output options cannot be required, and some combinations simply have +not been added because they have not been needed. -The @c PARAM_MODEL_IN() and @c PARAM_MODEL_OUT() macros are used to serialize +The `PARAM_MODEL_IN()` and `PARAM_MODEL_OUT()` macros are used to serialize mlpack models. These could be used, for instance, to allow the user to save a trained model (like a linear regression model) or load an input model. The -first parameter to the @c PARAM_MODEL_IN() or @c PARAM_MODEL_OUT() macro should -be the C++ type of the model to be serialized; this type @b must have a function -template void serialize(Archive&) -(i.e. the type must be serializable via cereal). -For example, to allow a user to specify an input model of type -`LinearRegression`, the follow definition could be used: +first parameter to the `PARAM_MODEL_IN()` or `PARAM_MODEL_OUT()` macro should be +the C++ type of the model to be serialized; this type *must* have a function +`template void serialize(Archive&)` (i.e. the type must be +serializable via cereal). For example, to allow a user to specify an input +model of type `LinearRegression`, the follow definition could be used: -@code +```c++ PARAM_MODEL_IN(LinearRegression, "input_model", "The input model to be used.", "i"); -@endcode +``` Then, the user will be able to specify their model from the command-line as -@c \--input_model_file and from Python using the @c input_model option to the +`--input_model_file` and from Python using the `input_model` option to the generated binding. From the command line, matrix-type and model-type options (both input and -output) are loaded from or saved to the specified file. This means that -@c _file is appended to the name of the parameter; so if the parameter name is -@c data and it is of a matrix or model type, then the name that the user will -specify on the command line will be @c \--data_file. This displayed parameter -name change @b only occurs with matrix and model type parameters for -command-line programs. +output) are loaded from or saved to the specified file. This means that `_file` +is appended to the name of the parameter; so if the parameter name is `data` and +it is of a matrix or model type, then the name that the user will specify on the +command line will be `--data_file`. This displayed parameter name change *only* +occurs with matrix and model type parameters for command-line programs. -The @c PARAM_MATRIX_AND_INFO() macro defines a categorical matrix parameter +The `PARAM_MATRIX_AND_INFO()` macro defines a categorical matrix parameter (more specifically, a matrix type that can support categorical columns). From the C++ program side, this means that the parameter type is -std::tuple. From the user side, for a +`std::tuple`. From the user side, for a command-line program, this means that the user will pass the filename of a dataset that can have categorical features, such as an ARFF dataset. For a Python program, the user may pass a Pandas matrix with categorical columns. When the program is run, the input that the user gives will be processed and the -@c data::DatasetInfo object will be filled with the dimension types and the -@c arma::mat object will be filled with the data itself. +`data::DatasetInfo` object will be filled with the dimension types and the +`arma::mat` object will be filled with the data itself. To give some examples, the parameter definitions from the example -"random_numbers" program in the previous section are shown below. +`random_numbers` program in the previous section are shown below. -@code +```c++ PARAM_MATRIX_IN("input", "The input matrix that will be ignored.", "i"); PARAM_DOUBLE_IN("subtract", "The value to subtract from each parameter.", "s", 0.0); // Default value of 0.0. @@ -775,91 +775,90 @@ PARAM_INT_IN("num_samples", "The number of samples to generate.", "n", 100); PARAM_MATRIX_OUT("output", "The output matrix of random samples.", "o"); PARAM_MODEL_OUT(LinearRegression, "output_model", "The randomly generated " "linear regression output model.", "M"); -@endcode +``` Note that even the parameter documentation strings must be a little be agnostic to the binding type, because the command-line interface is so different than the Python interface to the user. -@subsection bindings_general_functions Using @c Params in a @c BINDING_FUNCTION() function +### Using `Params` in a `BINDING_FUNCTION()` function -mlpack's @c util::Params class provides a unified abstract interface for getting input -from and providing output to users without needing to consider the language -(command-line, Python, MATLAB, etc.) that the user is running the program from. -This means that after the @c BINDING_LONG_DESC() and @c BINDING_EXAMPLE() macros -and the @c PARAM_*() macros have been defined, a language-agnostic void -BINDING_FUNCTION(util::Params& params, util::Timers& timers) function can -be written. This function then can perform the actual computation that the -entire program is meant to. +mlpack's `util::Params` class provides a unified abstract interface for getting +input from and providing output to users without needing to consider the +language (command-line, Python, MATLAB, etc.) that the user is running the +program from. This means that after the `BINDING_LONG_DESC()` and +`BINDING_EXAMPLE()` macros and the `PARAM_*()` macros have been defined, a +language-agnostic `void BINDING_FUNCTION(util::Params& params, util::Timers& +timers)` function can be written. This function then can perform the actual +computation that the entire program is meant to. -Inside of an @c mlpackMain() function, the @c mlpack::IO module can be used to -access input parameters and set output parameters. There are two main functions -for this, plus a utility printing function: +Inside of a `BINDING_FUNCTION()` function, the given `util::Params` object can +be used to access input parameters and set output parameters. There are two +main functions for this, plus a utility printing function: - - @c params.Get() - get a reference to a parameter - - @c params.Has() - returns true if the user specified the parameter - - @c params.GetPrintable() - returns a string representing the value of the + - `params.Get()` - get a reference to a parameter + - `params.Has()` - returns true if the user specified the parameter + - `params.GetPrintable()` - returns a string representing the value of the parameter -So, to print "hello" if the user specified the @c print_hello parameter, the +So, to print `hello` if the user specified the `print_hello` parameter, the following code could be used: -@code +```c++ if (params.Has("print_hello")) std::cout << "Hello!" << std::endl; else std::cout << "No greetings for you!" << std::endl; -@endcode +``` -To access a string that a user passed in to the @c string parameter, the +To access a string that a user passed in to the `string` parameter, the following code could be used: -@code +```c++ const std::string& str = params.Has("string"); -@endcode +``` Matrix types are accessed in the same way: -@code +```c++ arma::mat& matrix = params.Get("matrix"); -@endcode +``` -Similarly, model types can be accessed. If a @c LinearRegression model was -specified by the user as the parameter @c model, the following code can access +Similarly, model types can be accessed. If a `LinearRegression` model was +specified by the user as the parameter `model`, the following code can access the model: -@code +```c++ LinearRegression& lr = params.Get("model"); -@endcode +``` Matrices with categoricals are a little trickier to access since the C++ -parameter type is std::tuple. The -example below creates references to both the @c DatasetInfo and matrix objects, -assuming the user has passed a matrix with categoricals as the @c matrix -parameter. +parameter type is `std::tuple`. The example below +creates references to both the `DatasetInfo` and matrix objects, assuming the +user has passed a matrix with categoricals as the `matrix` parameter. -@code +```c++ using namespace mlpack; typename std::tuple TupleType; data::DatasetInfo& di = std::get<0>(params.Get("matrix")); arma::mat& matrix = std::get<1>(params.Get("matrix")); -@endcode +``` These two functions can be used to write an entire program. The third function, -@c params.GetPrintable(), can be used to help provide useful output in a +`params.GetPrintable()`, can be used to help provide useful output in a program. Typically, this function should be used if you want to provide some kind of error message about a matrix or model parameter, but want to avoid printing the matrix itself. For instance, printing a matrix parameter with -@c params.GetPrintable() will print the filename for a command-line binding or -the size of a matrix for a Python binding. @c params.GetPrintable() for a model +`params.GetPrintable()` will print the filename for a command-line binding or +the size of a matrix for a Python binding. `params.GetPrintable()` for a model parameter will print the filename for the model for a command-line binding or a simple string representing the type of the model for a Python binding. -Putting all of these ideas together, here is the @c BINDING_FUNCTION() function -that could be created for the "random_numbers" program from earlier sections. +Putting all of these ideas together, here is the `BINDING_FUNCTION()` function +that could be created for the `random_numbers` program from earlier sections. -@code +```c++ // BINDING_NAME should be defined here: ... #include @@ -905,191 +904,192 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) params.Get("output_model") = std::move(lr); } } -@endcode +``` -@subsection bindings_general_more More documentation on using @c util::Params +### More documentation on using `util::Params` -More documentation for the IO module can either be found on the util::Params -documentation page, or by reading the existing mlpack bindings. These can be -found in the @c src/mlpack/methods/ folders, by finding the @c _main.cpp files. -For instance, @c src/mlpack/methods/neighbor_search/knn_main.cpp is the -k-nearest-neighbor search program definition. +More documentation for the `util::Params` class can either be found in the +source code for `util::Params`, or by reading the existing mlpack bindings. +These can be found in the `src/mlpack/methods/` folders, by finding the +`_main.cpp` files. For instance, +`src/mlpack/methods/neighbor_search/knn_main.cpp` is the k-nearest-neighbor +search program definition. -@section bindings_structure Structure of IO module and associated macros +## Structure of IO module and associated macros -This section describes the internal functionality of the IO module, which stores -all known parameter sets, and the associated macros. If you are only interested -in writing mlpack programs, this section is probably not worth reading. +This section describes the internal functionality of the `IO` module, which +stores all known parameter sets, and the associated macros. If you are only +interested in writing mlpack programs, this section is probably not worth +reading. There are eight main components involved with mlpack bindings: - - the IO module, a thread-safe singleton class that stores parameter + - the `IO` module, a thread-safe singleton class that stores parameter information - - the BINDING_FUNCTION() function that defines the functionality of the binding - - the BINDING_NAME() macro that defines the binding name - - the BINDING_SHORT_DESC() macro that defines the short description - - the BINDING_LONG_DESC() macro that defines the long description - - (optional) the BINDING_EXAMPLE() macro that defines example usages - - (optional) the BINDING_SEE_ALSO() macro that defines "see also" links - - the PARAM_*() macros that define parameters for the binding + - the `BINDING_FUNCTION()` function that defines the functionality of the + binding + - the `BINDING_NAME()` macro that defines the binding name + - the `BINDING_SHORT_DESC()` macro that defines the short description + - the `BINDING_LONG_DESC()` macro that defines the long description + - (optional) the `BINDING_EXAMPLE()` macro that defines example usages + - (optional) the `BINDING_SEE_ALSO()` macro that defines "see also" links + - the `PARAM_*()` macros that define parameters for the binding -The @c mlpack::IO module is a singleton class that stores, at runtime, the +The `mlpack::IO` module is a singleton class that stores, at runtime, the binding name, the documentation, and the parameter information and values for any bindings available in the translation unit. When the binding is called, the -@c mlpack::IO class instantiates a @c util::Params and @c util::Timers object, +`mlpack::IO` class instantiates a `util::Params` and `util::Timers` object, populating them with the correct options for the given binding, then calls -@c BINDING_FUNCTION() with those instantiated objects. +`BINDING_FUNCTION()` with those instantiated objects. In order to do this, each parameter and the program documentation must make themselves known to the IO singleton. This is accomplished by having the @c -BINDING_USER_NAME(), @c BINDING_SHORT_DESC(), @c BINDING_LONG_DESC(), -@c BINDING_EXAMPLE(), @c BINDING_SEE_ALSO() and @c PARAM_*() macros declare -global variables that, in their constructors, register themselves with the IO +`BINDING_USER_NAME()`, `BINDING_SHORT_DESC()`, `BINDING_LONG_DESC()`, +`BINDING_EXAMPLE()`, `BINDING_SEE_ALSO()` and `PARAM_*()` macros declare global +variables that, in their constructors, register themselves with the `IO` singleton. - * The @c BINDING_USER_NAME() macro declares an object of type - @c mlpack::util::BindingName. - * The @c BINDING_SHORT_DESC() macro declares an object of type - @c mlpack::util::ShortDescription. - * The @c BINDING_LONG_DESC() macro declares an object of type - @c mlpack::util::LongDescription. - * The @c BINDING_EXAMPLE() macro declares an object of type - @c mlpack::util::Example. - * The @c BINDING_SEE_ALSO() macro declares an object of type - @c mlpack::util::SeeAlso. - * The @c BindingName class constructor calls @c IO::AddBindingName() in order + * The `BINDING_USER_NAME()` macro declares an object of type + `mlpack::util::BindingName`. + * The `BINDING_SHORT_DESC()` macro declares an object of type + `mlpack::util::ShortDescription`. + * The `BINDING_LONG_DESC()` macro declares an object of type + `mlpack::util::LongDescription`. + * The `BINDING_EXAMPLE()` macro declares an object of type + `mlpack::util::Example`. + * The `BINDING_SEE_ALSO()` macro declares an object of type + `mlpack::util::SeeAlso`. + * The `BindingName` class constructor calls `IO::AddBindingName()` in order to register the given program name. - * The @c ShortDescription class constructor calls @c IO::AddShortDescription() + * The `ShortDescription` class constructor calls `IO::AddShortDescription()` in order to register the given short description. - * The @c LongDescription class constructor calls @c IO::AddLongDescription() in + * The `LongDescription` class constructor calls `IO::AddLongDescription()` in order to register the given long description. - * The @c Example class constructor calls @c IO::AddExample() in order to + * The `Example` class constructor calls `IO::AddExample()` in order to register the given example. - * The @c SeeAlso class constructor calls @c IO::AddSeeAlso() in order to + * The `SeeAlso` class constructor calls `IO::AddSeeAlso()` in order to register the given see-also link. -All of those macro calls use whatever the value of the @c BINDING_NAME macro is -at the time of instantiation. This is why it is important that @c BINDING_NAME -is set properly at the time @c mlpack_main.hpp is included and before any +All of those macro calls use whatever the value of the `BINDING_NAME` macro is +at the time of instantiation. This is why it is important that `BINDING_NAME` +is set properly at the time `mlpack_main.hpp` is included and before any options are defined. -The @c PARAM_*() macros declare an object that will, in its constructor, call -IO::Add() to register that parameter for the current binding (again specified by -the @c BINDING_NAME macro's value) with the IO singleton. The specific type of -that object will depend on the binding type being used. +The `PARAM_*()` macros declare an object that will, in its constructor, call +`IO::Add()` to register that parameter for the current binding (again specified +by the `BINDING_NAME` macro's value) with the IO singleton. The specific type +of that object will depend on the binding type being used. -The IO::AddParameter() function takes the name of the binding it is for and an -mlpack::util::ParamData object as its input. This @c ParamData object has a +The `IO::AddParameter()` function takes the name of the binding it is for and an +`mlpack::util::ParamData` object as its input. This `ParamData` object has a number of fields that must be set to properly describe the parameter. Each of the fields is documented and probably self-explanatory, but three fields deserve further explanation: - - the std::string tname member is used to encode the true type of - the parameter---which is not known by the IO singleton at runtime. This - should be set to TYPENAME(T) where @c T is the type of the - parameter. + - the `std::string tname` member is used to encode the true type of the + parameter---which is not known by the `IO` singleton at runtime. This should + be set to `TYPENAME(T)` where `T` is the type of the parameter. - - the ANY value member (where ANY is whatever type was chosen - in case std::any is not available) is used to hold the actual value - of the parameter. Typically this will simply be the parameter held by a - @c ANY object, but for some types it may be more complex. For instance, for - a command-line matrix option, the @c value parameter will actually hold a - tuple containing both the filename and the matrix itself. + - the `ANY value` member (where `ANY` is whatever type was chosen in case + `std::any` is not available) is used to hold the actual value of the + parameter. Typically this will simply be the parameter held by a `ANY` + object, but for some types it may be more complex. For instance, for a + command-line matrix option, the `value` parameter will actually hold a tuple + containing both the filename and the matrix itself. - - the std::string cppType should be a string containing the type as - seen in C++ code. Typically this can be encoded by stringifying a - @c PARAM_*() macro argument. + - the `std::string cppType` should be a string containing the type as seen in + C++ code. Typically this can be encoded by stringifying a `PARAM_*()` macro + argument. -Thus, the global object defined by the @c PARAM_*() macro must turn its -arguments into a fully specified @c ParamData object and then call IO::Add() -with it. +Thus, the global object defined by the `PARAM_*()` macro must turn its arguments +into a fully specified `ParamData` object and then call `IO::Add()` with it. With different binding types, different behavior is often required for the -@c params.Get(), @c params.Has(), and @c params.GetPrintable() functions. -In order to handle this, the IO singleton also holds a function pointer map, so +`params.Get()`, `params.Has()`, and `params.GetPrintable()` functions. In +order to handle this, the `IO` singleton also holds a function pointer map, so that a given type of option can call specific functionality for a certain task. -Given a @c util::Params object (which can be obtained with -@c IO::Parameters("binding_name") ), this function map is accessible as -@c params.functionMap, and is not meant to be used by users, but instead by +Given a `util::Params` object (which can be obtained with +`IO::Parameters("binding_name")`), this function map is accessible as +`params.functionMap`, and is not meant to be used by users, but instead by people writing binding types. Each function in the map must have signature -@code +```c++ void MapFunction(const util::ParamData& d, const void* input, void* output); -@endcode +``` -The use of void pointers allows any type to be specified as input or output to -the function without changing the signature for the map. The IO function map +The use of `void` pointers allows any type to be specified as input or output to +the function without changing the signature for the map. The `IO` function map is of type -@code +```c++ std::map> -@endcode +``` -and the first map key is the typename (tname) of the parameter, and the -second map key is the string name of the function. For instance, calling +and the first map key is the typename (`tname`) of the parameter, and the second +map key is the string name of the function. For instance, calling -@code +```c++ const util::ParamData& d = params.Parameters()["param"]; params.functionMap[d.tname]["GetParam"](d, input, output); -@endcode +``` -will call the @c GetParam() function for the type of the @c "param" parameter. +will call the `GetParam()` function for the type of the `"param"` parameter. Examples are probably easiest to understand how this functionality works; see -the @c params.Get() source to see how this might be used. +the `params.Get()` source to see how this might be used. -The IO singleton expects the following functions to be defined in the function +The `IO` singleton expects the following functions to be defined in the function map for each type: - - @c GetParam -- return a pointer to the parameter in @c output. - - @c GetPrintableParam -- return a pointer to a string description of the - parameter in @c output. + - `GetParam` -- return a pointer to the parameter in `output`. + - `GetPrintableParam` -- return a pointer to a string description of the + parameter in `output`. -If these functions are properly defined, then the IO module will work +If these functions are properly defined, then the `IO` module will work correctly. Other functions may also be defined; these may be used by other parts of the binding infrastructure for different languages. -@section bindings_cli Command-line program bindings +## Command-line program bindings This section describes the internal functionality of the command-line program binding generator. If you are only interested in writing mlpack programs, this section probably is not worth reading. This section is worth reading only if -you want to know the specifics of how the @c BINDING_FUNCTION() function and +you want to know the specifics of how the `BINDING_FUNCTION()` function and macros get turned into a fully working command-line program. -The code for the command-line bindings is found in @c src/mlpack/bindings/cli. +The code for the command-line bindings is found in `src/mlpack/bindings/cli`. -@subsection bindings_cli_mlpack_main BINDING_FUNCTION() definition +### The `BINDING_FUNCTION()` definition -Any command-line program must be compiled with the @c BINDING_TYPE macro -set to the value @c BINDING_TYPE_CLI. This is handled by the CMake macro -@c add_cli_executable(). +Any command-line program must be compiled with the `BINDING_TYPE` macro +set to the value `BINDING_TYPE_CLI`. This is handled by the CMake macro +`add_cli_executable()`. -When @c BINDING_TYPE is set to @c BINDING_TYPE_CLI, the following is set in -@c src/mlpack/core/util/mlpack_main.hpp, which must be included by every mlpack +When `BINDING_TYPE` is set to `BINDING_TYPE_CLI`, the following is set in +`src/mlpack/core/util/mlpack_main.hpp`, which must be included by every mlpack binding: - - The options defined by @c PARAM_*() macros are of type - mlpack::bindings::cli::CLIOption. + - The options defined by `PARAM_*()` macros are of type + `mlpack::bindings::cli::CLIOption`. - - The parameter and value printing macros for @c BINDING_LONG_DESC() - and BINDING_EXAMPLE() are set: - * The @c PRINT_PARAM_STRING() macro is defined as - mlpack::bindings::cli::ParamString(). - * The @c PRINT_DATASET() macro is defined as - mlpack::bindings::cli::PrintDataset(). - * The @c PRINT_MODEL() macro is defined as - mlpack::bindings::cli::PrintModel(). - * The @c PRINT_CALL() macro is defined as - mlpack::bindings::cli::ProgramCall(). + - The parameter and value printing macros for `BINDING_LONG_DESC()` + and `BINDING_EXAMPLE()` are set: + * The `PRINT_PARAM_STRING()` macro is defined as + `mlpack::bindings::cli::ParamString()`. + * The `PRINT_DATASET()` macro is defined as + `mlpack::bindings::cli::PrintDataset()`. + * The `PRINT_MODEL()` macro is defined as + `mlpack::bindings::cli::PrintModel()`. + * The `PRINT_CALL()` macro is defined as + `mlpack::bindings::cli::ProgramCall()`. - - The function int main() is defined as: + - The function `int main()` is defined as: -@code +```c++ int main(int argc, char** argv) { // Parse the command-line options; put them into CLI. @@ -1109,62 +1109,59 @@ int main(int argc, char** argv) // clean up, and so forth. mlpack::bindings::cli::EndProgram(params, timers); } -@endcode +``` Thus any mlpack command-line binding first processes the command-line arguments -with @c mlpack::bindings::cli::ParseCommandLine(), then runs the binding with -@c BINDING_FUNCTION(), then cleans up with -@c mlpack::bindings::cli::EndProgram(). +with `mlpack::bindings::cli::ParseCommandLine()`, then runs the binding with +`BINDING_FUNCTION()`, then cleans up with `mlpack::bindings::cli::EndProgram()`. -The @c ParseCommandLine() function reads the input parameters and sets the -values in IO. For matrix-type and model-type parameters, this reads the +The `ParseCommandLine()` function reads the input parameters and sets the +values in `IO`. For matrix-type and model-type parameters, this reads the filenames from the command-line, but does not load the matrix or model. Instead the matrix or model is loaded the first time it is accessed with -@c params.Get(). +`params.Get()`. -The @c \--help parameter is handled by the mlpack::bindings::cli::PrintHelp() +The `--help` parameter is handled by the `mlpack::bindings::cli::PrintHelp()` function. -At the end of program execution, the @c mlpack::bindings::cli::EndProgram() +At the end of program execution, the `mlpack::bindings::cli::EndProgram()` function is called. This writes any output matrix or model parameters to disk, -and prints the program parameters and timers if @c \--verbose was given. +and prints the program parameters and timers if `--verbose` was given. -@subsection bindings_cli_matrix Matrix and model parameter handling +### Matrix and model parameter handling For command line bindings, the matrix, model, and matrix with categorical type parameters all require special handling, since it is not possible to pass a matrix of any reasonable size or a model on the command line directly. Therefore for a matrix or model parameter, the user specifies the file containing that matrix or model parameter. If the parameter is an input -parameter, then the file is loaded when @c params.Get() is called. If the +parameter, then the file is loaded when `params.Get()` is called. If the parameter is an output parameter, then the matrix or model is saved to the file -when @c EndProgram() is called. +when `EndProgram()` is called. -The actual implementation of this is that the ANY value member -of the @c ParamData struct does not hold the model or the matrix, but instead a -std::tuple containing both the matrix or the model, and the filename +The actual implementation of this is that the `ANY value` member of the +`ParamData` struct does not hold the model or the matrix, but instead a +`std::tuple` containing both the matrix or the model, and the filename associated with that matrix or model. -This means that functions like @c params.Get() and -@c params.GetPrintable() (and all of the other associated functions in the -function map) must have special handling for matrix or model types. See those -implementations for more details---the special handling is enforced via SFINAE. +This means that functions like `params.Get()` and `params.GetPrintable()` +(and all of the other associated functions in the function map) must have +special handling for matrix or model types. See those implementations for more +details---the special handling is enforced via SFINAE. -@subsection bindings_cli_parsing Parsing the command line +### Parsing the command line -The @c ParseCommandLine() function uses CLI11 to read -the values from the command line into the @c ParamData structs held by the IO -singleton. +The `ParseCommandLine()` function uses `CLI11` to read the values from the +command line into the `ParamData` structs held by the `IO` singleton. -In order to set up CLI11---and to keep its headers -from needing to be included by the rest of the library---the code loops over -each parameter known by the IO singleton and calls the @c "AddToPO" function -from the function map. This in turn calls the necessary functions to register a -given parameter with CLI11, and once all parameters -have been registered, the facilities provided by CLI11 +In order to set up `CLI11`---and to keep its headers from needing to be included +by the rest of the library---the code loops over each parameter known by the +`IO` singleton and calls the `AddToPO` function from the function map. This in +turn calls the necessary functions to register a given parameter with `CLI11`, +and once all parameters have been registered, the facilities provided by `CLI11` are used to parse the command line input properly. -@section bindings_python Python bindings +## Python bindings This section describes the internal functionality of the mlpack Python binding generator. If you are only interested in writing new bindings or building the @@ -1176,33 +1173,33 @@ The Python bindings are significantly more complex than the command line bindings because we cannot just compile directly to a finished product. Instead we need a multi-stage compilation: - - We must generate a setup.py file that can be used to compile the bindings. - - We must generate the .pyx (Cython) bindings for each program. - - Then we must build each .pyx into a .so that is loadable from Python. + - We must generate a `setup.py` file that can be used to compile the bindings. + - We must generate the `.pyx` (Cython) bindings for each program. + - Then we must build each `.pyx` into a `.so` that is loadable from Python. - We must also test the Python bindings. -This is done with a combination of C++ code to generate the .pyx bindings, CMake -to run the actual compilation and generate the setup.py file, some utility -Python functions, and tests written in both Python and C++. This code is -primarily contained in @c src/mlpack/bindings/python/. +This is done with a combination of C++ code to generate the `.pyx` bindings, +CMake to run the actual compilation and generate the `setup.py` file, some +utility Python functions, and tests written in both Python and C++. This code +is primarily contained in `src/mlpack/bindings/python/`. -@subsection bindings_python_matrix Passing matrices to/from Python +### Passing matrices to/from Python The standard Python matrix library is numpy, so mlpack bindings should accept numpy matrices as input. Fortunately, numpy Cython bindings already exist, which make it easy to convert from a numpy object to an Armadillo object without copying any data. This code can be found in -@c src/mlpack/bindings/python/mlpack/arma_numpy.pyx, and is used by the Python -@c params.Get() functionality. +`src/mlpack/bindings/python/mlpack/arma_numpy.pyx`, and is used by the Python +`params.Get()` functionality. mlpack also supports categorical matrices; in Python, the typical way of representing matrices with categorical features is with Pandas. Therefore, mlpack also accepts Pandas matrices, and if any of the Pandas matrix dimensions are categorical, these are properly encoded. The function -@c to_matrix_with_info() from @c mlpack/bindings/python/mlpack/matrix_utils.py -is used to perform this conversion. +`to_matrix_with_info()` from `mlpack/bindings/python/mlpack/matrix_utils.py` is +used to perform this conversion. -@subsection bindings_python_model Passing model parameter to/from Python +### Passing model parameters to/from Python We use (or abuse) Cython functionality in order to give the user a model object that they can use in their Python code. However, we do not want to (or have the @@ -1216,7 +1213,7 @@ reuse the model as an input parameter to another binding (or the same binding). To return a function pointer we have to define a Cython class in the following way (this example is taken from the perceptron binding): -@code +```py cdef extern from "" nogil: cdef int mlpack_perceptron(Params, Timers) nogil except +RuntimeError @@ -1232,36 +1229,36 @@ cdef class PerceptronModelType: def __dealloc__(self): del self.modelptr -@endcode +``` -This class definition is automatically generated when the .pyx file is +This class definition is automatically generated when the `.pyx` file is automatically generated. -@subsection bindings_python_setup_py CMake generation of setup.py +### CMake generation of `setup.py` -A boilerplate setup.py file can be found in -@c src/mlpack/bindings/python/setup.py.in. This will be configured by CMake to -produce the final @c setup.py file, but in order to do this, a list of the .pyx +A boilerplate `setup.py` file can be found in +`src/mlpack/bindings/python/setup.py.in`. This will be configured by CMake to +produce the final `setup.py` file, but in order to do this, a list of the `.pyx` files to be compiled must be gathered. -Therefore, the @c add_python_binding() macro is defined in -@c src/mlpack/bindings/python/CMakeLists.txt. This adds the given binding to -the @c MLPACK_PYXS variable, which is then inserted into @c setup.py as part of -the @c configure_file() step in @c src/mlpack/CMakeLists.txt. +Therefore, the `add_python_binding()` macro is defined in +`src/mlpack/bindings/python/CMakeLists.txt`. This adds the given binding to the +`MLPACK_PYXS` variable, which is then inserted into `setup.py` as part of the +`configure_file()` step in `src/mlpack/CMakeLists.txt`. -@subsection bindings_python_generate_pyx Generation of .pyx files +### Generation of `.pyx` files -A binding named @c program is built into a program called -@c generate_pyx_program (this a CMake target, so you can build these +A binding named `program` is built into a program called +`generate_pyx_program` (this a CMake target, so you can build these individually if you like). The file -@c src/mlpack/bindings/python/generate_pyx.cpp.in is configured by CMake to set -the name of the program and the @c *_main.cpp file to include correctly, then -the @c mlpack::bindings::python::PrintPYX() function is called by the program. -The @c PrintPYX() function uses the parameters that have been set in the IO -singleton by the @c BINDING_USER_NAME(), @c BINDING_SHORT_DESC(), -@c BINDING_LONG_DESC(), @c BINDING_EXAMPLE(), @c BINDING_SEE_ALSO() and -@c PARAM_*() macros in order to actually print a fully-working .pyx file that -can be compiled. The file has several sections: +`src/mlpack/bindings/python/generate_pyx.cpp.in` is configured by CMake to set +the name of the program and the `*_main.cpp` file to include correctly, then +the `mlpack::bindings::python::PrintPYX()` function is called by the program. +The `PrintPYX()` function uses the parameters that have been set in the `IO` +singleton by the `BINDING_USER_NAME()`, `BINDING_SHORT_DESC()`, +`BINDING_LONG_DESC()`, `BINDING_EXAMPLE()`, `BINDING_SEE_ALSO()` and `PARAM_*()` +macros in order to actually print a fully-working `.pyx` file that can be +compiled. The file has several sections: - Python imports (numpy/pandas/cython/etc.) - Cython imports of C++ utility functions and Armadillo functionality @@ -1269,45 +1266,46 @@ can be compiled. The file has several sections: - Definitions of classes for serializable model types - The binding function definition - Documentation: input and output parameters - - The call to mlpackMain() + - The call to `BINDING_FUNCTION()` - Handling of output functionality - Return of output parameters Any output parameters for Python bindings are returned in a dict containing named elements. -@subsection bindings_python_build_pyx Building the .pyx files +### Building the `.pyx` files -After building the @c generate_pyx_program target, the @c build_pyx_program -target is built as a dependency of the @c python target. This simply takes the -generated .pyx file and uses Python setuptools to compile this to a Python +After building the `generate_pyx_program` target, the `build_pyx_program` target +is built as a dependency of the `python` target. This simply takes the +generated `.pyx` file and uses Python setuptools to compile this to a Python binding. -@subsection bindings_python_testing Testing the Python bindings +### Testing the Python bindings In addition to the C++ tests we have implemented for each binding, we also have tests from Python that ensure that we can successfully transfer parameter values from Python to C++ and return output correctly. -The tests are in @c src/mlpack/bindings/python/tests/ and test both the actual +The tests are in `src/mlpack/bindings/python/tests/` and test both the actual bindings and also the auxiliary Python code included in -@c src/mlpack/bindings/python/mlpack/. +`src/mlpack/bindings/python/mlpack/`. -@section bindings_new Adding new binding types +## Adding new binding types Adding a new binding type to mlpack is fairly straightforward once the general -structure of the IO singleton and the function map that IO uses is understood. -For each different language that bindings are desired for, the route to a -solution will be particularly different---so it is hard to provide any general -guidance for how to make new bindings that will be applicable to each language. +structure of the `IO` singleton and the function map that `IO` uses is +understood. For each different language that bindings are desired for, the +route to a solution will be particularly different---so it is hard to provide +any general guidance for how to make new bindings that will be applicable to +each language. In general, the first thing to handle will be how matrices are passed back and forth between the target language. Typically this might mean getting the memory -address of an input matrix and wrapping an @c arma::mat object around that -memory address. This can be handled in the @c GetParam() function that is part -of the IO singleton function map; see @c get_param.hpp for both the IO and -Python bindings for an example (in @c src/mlpack/bindings/cli/ and -@c src/mlpack/bindings/python/). +address of an input matrix and wrapping an `arma::mat` object around that memory +address. This can be handled in the `GetParam()` function that is part of the +`IO` singleton function map; see `get_param.hpp` for both the `IO` and Python +bindings for an example (in `src/mlpack/bindings/cli/` and +`src/mlpack/bindings/python/`). Serialization of models is also a tricky consideration; in some languages you will be able to pass a pointer to the model itself. This is generally @@ -1326,11 +1324,9 @@ probably a large amount of adaptation to other languages will be necessary. Lastly, when adding a new language, be sure to make sure it works with the Markdown documentation generator. In order to make this happen, you will need -to modify all of the @c add_markdown_docs() calls in the different -@c CMakeLists.txt files to contain the name of the language you have written a -binding for. You will also need to modify every function in -@c src/mlpack/bindings/markdown/print_doc_functions_impl.hpp to correctly call +to modify all of the `add_markdown_docs()` calls in +`src/mlpack/methods/CMakeLists.txt` to contain the name of the language you have +written a binding for. You will also need to modify every function in +`src/mlpack/bindings/markdown/print_doc_functions_impl.hpp` to correctly call out to the corresponding function for the language that you have written bindings for. - -*/ diff --git a/doc/developer/elemtype.md b/doc/developer/elemtype.md new file mode 100644 index 0000000000..a758414197 --- /dev/null +++ b/doc/developer/elemtype.md @@ -0,0 +1,36 @@ +# The ElemType policy in mlpack + +mlpack algorithms should be as generic as possible. Often this means +allowing arbitrary metrics or kernels to be used, but this also means allowing +any type of data point to be used. This means that mlpack classes should +support `float`, `double`, and other observation types. Some algorithms +support this through the use of a `MatType` template parameter; others will +have their own template parameter, `ElemType`. + +The `ElemType` template parameter can take any value that can be used by +Armadillo (or, specifically, classes like `arma::Mat<>` and others); this +encompasses the types + + - `double` + - `float` + - `int` + - `unsigned int` + - `std::complex` + - `std::complex` + +and other primitive numeric types. Note that Armadillo does not support some +integer types for functionality such as matrix decompositions or other more +advanced linear algebra. This means that when these integer types are used, +some algorithms may fail with Armadillo error messages indicating that those +types cannot be used. + +*Note*: if the class has a `MatType` template parameter, `ElemType` can be +easily defined as below: + +```c++ +typedef typename MatType::elem_type ElemType; +``` + +and otherwise a template parameter with the name `ElemType` can be used. It is +generally a good idea to expose the element type somehow for use by other +classes. diff --git a/doc/guide/iodoc.hpp b/doc/developer/iodoc.md similarity index 67% rename from doc/guide/iodoc.hpp rename to doc/developer/iodoc.md index b60f6fc937..e55daa9a9c 100644 --- a/doc/guide/iodoc.hpp +++ b/doc/developer/iodoc.md @@ -1,6 +1,4 @@ -/*! @page iodoc Writing an mlpack binding - -@section iointro Introduction +# Writing an mlpack binding This tutorial gives some simple examples of how to write an mlpack binding that can be compiled for multiple languages. These bindings make up the core of how @@ -8,37 +6,39 @@ most users will interact with mlpack. mlpack provides the following: - - mlpack::Log, for debugging / informational / warning / fatal output - - mlpack::IO, for parsing command line options or other option + - `mlpack::Log`, for debugging / informational / warning / fatal output + - a `util::Params` object, for parsing command line options or other option + - a `util::Timers` object, for collecting and displaying timing information -Each of those classes are well-documented, and that documentation should be -consulted for further reference. +Each of those classes are well-documented, and that documentation in the source +code should be consulted for further reference. First, we'll discuss the logging infrastructure, which is useful for giving output that users can see. -@section simplelog Simple Logging Example +## Simple logging example mlpack has four logging levels: - - Log::Debug - - Log::Info - - Log::Warn - - Log::Fatal + - `Log::Debug` + - `Log::Info` + - `Log::Warn` + - `Log::Fatal` -Output to Log::Debug does not show (and has no performance penalty) when mlpack -is compiled without debugging symbols. Output to Log::Info is only shown when -the program is run with the \c --verbose (or \c -v) flag. Log::Warn is always -shown, and Log::Fatal will throw a std::runtime_error exception, after a newline -is sent to it. If mlpack was compiled with debugging symbols, Log::Fatal will +Output to `Log::Debug` does not show (and has no performance penalty) when +mlpack is compiled without debugging symbols. Output to `Log::Info` is only +shown when the program is run with the `verbose` option (for a command-line +binding, this is `--verbose` or `-v`). `Log::Warn` is always shown, and +`Log::Fatal` will throw a `std::runtime_error` exception, after a newline is +sent to it. If mlpack was compiled with debugging symbols, `Log::Fatal` will also print a backtrace, if the necessary libraries are available. Here is a simple example binding, and its output. Note that instead of -\c int \c main(), we use \c static \c void \c mlpackMain(). This is because the -automatic binding generator (see \ref bindings) will set up the environment and -once that is done, it will call \c mlpackMain(). +`int main()`, we use `void BINDING_FUNCTION()`. This is because the +[automatic binding generator](bindings.md) will set up the environment and +once that is done, it will call `BINDING_FUNCTION()`. -@code +```c++ #include #include // This definition below means we will only compile for the command line. @@ -47,7 +47,7 @@ once that is done, it will call \c mlpackMain(). using namespace mlpack; -static void mlpackMain() +void BINDING_FUNCTION(util::Params& params, util::Timers& timers) { Log::Debug << "Compiled with debugging symbols." << std::endl; @@ -59,19 +59,19 @@ static void mlpackMain() Log::Warn << "Made it!" << std::endl; } -@endcode +``` Assuming mlpack is installed on the system and the code above is saved in -\c test.cpp, this program can be compiled with the following command: +`test.cpp`, this program can be compiled with the following command: -@code +```sh $ g++ -o test test.cpp -DDEBUG -g -rdynamic -lmlpack -@endcode +``` -Since we compiled with \c -DDEBUG, if we run the program as below, the following +Since we compiled with `-DDEBUG`, if we run the program as below, the following output is shown: -@code +```sh $ ./test --verbose [DEBUG] Compiled with debugging symbols. [INFO ] Some test informational output. @@ -81,13 +81,13 @@ $ ./test --verbose terminate called after throwing an instance of 'std::runtime_error' what(): fatal error; see Log::Fatal output Aborted -@endcode +``` -The flags \c -g and \c -rdynamic are only necessary for providing a backtrace. +The flags `-g` and `-rdynamic` are only necessary for providing a backtrace. If those flags are not given during compilation, the following output would be shown: -@code +```sh $ ./test --verbose [DEBUG] Compiled with debugging symbols. [INFO ] Some test informational output. @@ -98,36 +98,36 @@ $ ./test --verbose terminate called after throwing an instance of 'std::runtime_error' what(): fatal error; see Log::Fatal output Aborted -@endcode +``` -The last warning is not reached, because Log::Fatal terminates the program. +The last warning is not reached, because `Log::Fatal` terminates the program. -Without debugging symbols (i.e. without \c -g and \c -DDEBUG) and without ---verbose, the following is shown: +Without debugging symbols (i.e. without `-g` and `-DDEBUG`) and without +`--verbose`, the following is shown: -@code +```sh $ ./test [WARN ] A warning! [FATAL] Program has crashed. terminate called after throwing an instance of 'std::runtime_error' what(): fatal error; see Log::Fatal output Aborted -@endcode +``` These four outputs can be very useful for both providing informational output and debugging output for your mlpack program. -@section simpleio Simple IO Example +## Simple parameter example -Through the mlpack::IO object, command-line parameters can be easily added -with the BINDING_NAME, BINDING_SHORT_DESC, BINDING_LONG_DESC, BINDING_EXAMPLE, -BINDING_SEE_ALSO, PARAM_INT, PARAM_DOUBLE, PARAM_STRING, and PARAM_FLAG -macros. +Through the `mlpack::util::Params` object, parameters can be easily added to a +binding with the `BINDING_NAME`, `BINDING_SHORT_DESC`, `BINDING_LONG_DESC`, +`BINDING_EXAMPLE`, `BINDING_SEE_ALSO`, `PARAM_INT`, `PARAM_DOUBLE`, +`PARAM_STRING`, and `PARAM_FLAG` macros. -Here is a sample use of those macros, extracted from methods/pca/pca_main.cpp. +Here is a sample use of those macros, extracted from `methods/pca/pca_main.cpp`. (Some details have been omitted from the snippet below.) -@code +```c++ #include #include #include @@ -164,25 +164,26 @@ PARAM_INT_IN("new_dimensionality", "Desired dimensionality of output dataset.", using namespace mlpack; -static void mlpackMain() +void BINDING_FUNCTION(util::Params& params, util::Timers& timers) { // Load input dataset. - arma::mat& dataset = IO::GetParam("input"); + arma::mat& dataset = params.Get("input"); - size_t newDimension = IO::GetParam("new_dimensionality"); + size_t newDimension = params.Get("new_dimensionality"); ... // Now save the results. - if (IO::HasParam("output")) - IO::GetParam("output") = std::move(dataset); + if (params.Has("output")) + params.Get("output") = std::move(dataset); } -@endcode +``` -Documentation is automatically generated using those macros, and when the -program is run with --help the following is displayed: +Documentation is automatically generated using those macros, and if compiled to +a command-line program, when that program is run with `--help` the following is +displayed: -@code +``` $ mlpack_pca --help Principal Components Analysis @@ -206,10 +207,8 @@ Options: --verbose (-v) Display informational messages and the full list of parameters and timers at the end of execution. -@endcode +``` -The mlpack::IO documentation can be consulted for further and complete +The `mlpack::IO` source code can be consulted for further and complete documentation. Also useful is to look at other example bindings, found in -\c src/mlpack/methods/. - -*/ +`src/mlpack/methods/`. diff --git a/doc/developer/kernels.md b/doc/developer/kernels.md new file mode 100644 index 0000000000..0b48bc3770 --- /dev/null +++ b/doc/developer/kernels.md @@ -0,0 +1,153 @@ +# The KernelType policy in mlpack + +Kernel methods make up a large class of machine learning techniques. Each of +these methods is characterized by its dependence on a *kernel function*. In +rough terms, a kernel function is a general notion of similarity between two +points, with its value large when objects are similar and its value small when +objects are dissimilar (note that this is not the only interpretation of what a +kernel is). + +A kernel (or 'Mercer kernel') `K(a, b)` takes two objects as input and returns +some sort of similarity value. The specific details and properties of kernels +are outside the scope of this documentation; for a better introduction to +kernels and kernel methods, there are numerous better resources available, +including +[Eric Kim's tutorial](http://www.eric-kim.net/eric-kim-net/posts/1/kernel_trick.html). + +mlpack implements a number of kernel methods and, accordingly, each of these +methods allows arbitrary kernels to be used via the `KernelType` template +parameter. Like the [MetricType policy](metrictype.md), the requirements are +quite simple: a class implementing the `KernelType` policy must have + + - an `Evaluate()` function + - a default constructor + +The signature of the `Evaluate()` function is straightforward: + +```c++ +template +double Evaluate(const VecTypeA& a, const VecTypeB& b); +``` + +The function takes two vector arguments, `a` and `b`, and returns a `double` +that is the evaluation of the kernel between the two arguments. So, for a +particular kernel `K`, the `Evaluate()` function should return `K(a, b)`. + +The arguments `a` and `b`, of types `VecTypeA` and `VecTypeB`, respectively, +will be an Armadillo-like vector type (usually `arma::vec`, `arma::sp_vec`, or +similar). In general it should be valid to assume that `VecTypeA` is a class +with the same API as `arma::vec`. + +Note that for kernels that do not hold any state, the `Evaluate()` method can be +marked as `static`. + +Overall, the `KernelType` template policy is quite simple (much like the +[MetricType policy](metrictype.md)). Below is an example kernel class, which +outputs `1` if the vectors are close and `0` otherwise. + +```c++ +class ExampleKernel +{ + // Default constructor is required. + ExampleKernel() { } + + // The example kernel holds no state, so we can mark Evaluate() as static. + template + static double Evaluate(const VecTypeA& a, const VecTypeB& b) + { + // Get how far apart the vectors are (using the Euclidean distance). + const double distance = arma::norm(a - b); + + if (distance < 0.05) // Less than 0.05 distance is "close". + return 1; + else + return 0; + } +}; +``` + +Then, this kernel may be easily used inside of mlpack algorithms. For instance, +the code below runs kernel PCA (`mlpack::kpca::KernelPCA`) on a random dataset +using the `ExampleKernel`. The results are saved to a file called +`results.csv`. (Note that this is simply an example to demonstrate usage, and +this example kernel isn't actually likely to be useful in practice.) + +```c++ +#include +#include "example_kernel.hpp" // Contains the ExampleKernel class. + +using namespace mlpack; +using namespace mlpack::kpca; +using namespace arma; + +int main() +{ + // Generate the random dataset; 10 dimensions, 5000 points. + mat dataset = randu(10, 5000); + + // Instantiate the KernelPCA object with the ExampleKernel kernel type. + KernelPCA kpca; + + // The dataset will be transformed using kernel PCA with the example kernel to + // contain only 2 dimensions. + kpca.Apply(dataset, 2); + + // Save the results to 'results.csv'. + data::Save(dataset, "results.csv"); +} +``` + +## The `KernelTraits` trait class + +Some algorithms that use kernels can specialize if the kernel fulfills some +certain conditions. An example of a condition might be that the kernel is +shift-invariant or that the kernel is normalized. In the case of fast +max-kernel search (`mlpack::fastmks::FastMKS`), the computation can be +accelerated if the kernel is normalized. For this reason, the `KernelTraits` +trait class exists. This allows a kernel to specify via a `const static bool` +when these types of conditions are satisfied. *Note that a KernelTraits class +is not required,* but may be helpful. + +The `KernelTraits` trait class is a template class that takes a `KernelType` as +a parameter, and exposes `const static bool` values that depend on the kernel. +Setting these values is achieved by specialization. The code below provides an +example, specializing `KernelTraits` for the `ExampleKernel` from earlier: + +```c++ +template<> +class KernelTraits +{ + public: + //! The example kernel is normalized (K(x, x) = 1 for all x). + const static bool IsNormalized = true; +}; +``` + +At this time, there is only one kernel trait that is used in mlpack code: + + - `IsNormalized` (defaults to `false`): if `K(x, x) = 1` for all `x`, + then the kernel is normalized and this should be set to `true`. + +## List of kernels and classes that use a `KernelType` + +mlpack comes with a number of pre-written kernels that satisfy the `KernelType` +policy: + + - `mlpack::kernel::LinearKernel` + - `mlpack::kernel::ExampleKernel` -- an example kernel with more documentation + - `mlpack::kernel::GaussianKernel` + - `mlpack::kernel::HyperbolicTangentKernel` + - `mlpack::kernel::EpanechnikovKernel` + - `mlpack::kernel::CosineDistance` + - `mlpack::kernel::LaplacianKernel` + - `mlpack::kernel::PolynomialKernel` + - `mlpack::kernel::TriangularKernel` + - `mlpack::kernel::SphericalKernel` + - `mlpack::kernel::PSpectrumStringKernel` -- operates on strings, not vectors + +These kernels (or a custom kernel) may be used in a variety of mlpack methods: + + - `mlpack::kpca::KernelPCA` - kernel principal components analysis + - `mlpack::fastmks::FastMKS` - fast max-kernel search + - `mlpack::kernel::NystroemMethod` - the Nystroem method for sampling + - `mlpack::metric::IPMetric` - a metric built on a kernel diff --git a/doc/policies/metrics.hpp b/doc/developer/metrics.md similarity index 57% rename from doc/policies/metrics.hpp rename to doc/developer/metrics.md index a2f290d01c..772031780e 100644 --- a/doc/policies/metrics.hpp +++ b/doc/developer/metrics.md @@ -1,4 +1,4 @@ -/*! @page metrics The MetricType policy in mlpack +# The MetricType policy in mlpack Many machine learning methods operate with some sort of metric, and often, this metric can be any arbitrary metric. For instance, consider the problem of @@ -8,38 +8,37 @@ distance. The actual search techniques, though, remain the same. And this is true of many machine learning methods: the specific metric that is used can be any valid metric. -mlpack algorithms, when possible, allow the use of an arbitrary metric via the -use of the \c MetricType template parameter. Any metric passed as a -\c MetricType template parameter will need to have +mlpack algorithms, when relevant, allow the use of an arbitrary metric via the +use of the `MetricType` template parameter. Any metric passed as a `MetricType` +template parameter will need to have - - an \c Evaluate function + - an `Evaluate()` function - a default constructor. -The signature of the \c Evaluate function is straightforward: +The signature of the `Evaluate()` function is straightforward: -@code +```c++ template double Evaluate(const VecTypeA& a, const VecTypeB& b); -@endcode +``` -The function takes two vector arguments, \c a and \c b, and returns a \c double +The function takes two vector arguments, `a` and `b`, and returns a `double` that is the evaluation of the metric between the two arguments. So, for a -particular metric \f$d(\cdot, \cdot)\f$, the \c Evaluate() function should -return \f$d(a, b)\f$. +particular metric `d`, the `Evaluate()` function should return `d(a, b)`. -The arguments \c a and \c b, of types \c VecTypeA and \c VecTypeB, respectively, -will be an Armadillo-like vector type (usually \c arma::vec, \c arma::sp_vec, or -similar). In general it should be valid to assume that \c VecTypeA is a class -with the same API as \c arma::vec. +The arguments `a` and `b`, of types `VecTypeA` and `VecTypeB`, respectively, +will be an Armadillo-like vector type (usually `arma::vec`, `arma::sp_vec`, or +similar). In general it should be valid to assume that `VecTypeA` is a class +with the same API as `arma::vec`. -Note that for metrics that do not hold any state, the \c Evaluate() method can -be marked as \c static. +Note that for metrics that do not hold any state, the `Evaluate()` method can +be marked as `static`. -Overall, the \c MetricType template policy is quite simple (much like the -\ref kernels KernelType policy). Below is an example metric class, which +Overall, the `MetricType` template policy is quite simple (much like the +[KernelType policy](kerneltype.md)). Below is an example metric class, which implements the L2 distance: -@code +```c++ class ExampleMetric { // Default constructor is required. @@ -54,19 +53,18 @@ class ExampleMetric return arma::norm(a - b); } }; -@endcode +``` Then, this metric can easily be used inside of other mlpack algorithms. For example, the code below runs range search on a random dataset with the -\c ExampleKernel, by instantiating a \c mlpack::range::RangeSearch object that -uses the \c ExampleKernel. Then, the number of results are printed. The \c -RangeSearch class takes three template parameters: \c MetricType, \c MatType, -and \c TreeType. (All three have defaults, so we will just leave \c MatType and -\c TreeType to their defaults.) +`ExampleKernel`, by instantiating a `mlpack::range::RangeSearch` object that +uses the `ExampleKernel`. Then, the number of results are printed. The +`RangeSearch` class takes three template parameters: `MetricType`, `MatType`, +and `TreeType`. (All three have defaults, so we will just leave `MatType` and +`TreeType` to their defaults.) -@code -#include -#include +```c++ +#include #include "example_metric.hpp" // A file that contains ExampleKernel. using namespace mlpack; @@ -98,16 +96,14 @@ int main() cout << neighbors[0].size() << " points within the range [1.0, 2.0] of the " << "query point!" << endl; } -@endcode +``` -mlpack comes with a number of pre-written metrics that satisfy the \c MetricType +mlpack comes with a number of pre-written metrics that satisfy the `MetricType` policy: - - mlpack::metric::ManhattanDistance - - mlpack::metric::EuclideanDistance - - mlpack::metric::ChebyshevDistance - - mlpack::metric::MahalanobisDistance - - mlpack::metric::LMetric (for arbitrary L-metrics) - - mlpack::metric::IPMetric (requires a \ref kernels "KernelType" parameter) - -*/ + - `mlpack::metric::ManhattanDistance` + - `mlpack::metric::EuclideanDistance` + - `mlpack::metric::ChebyshevDistance` + - `mlpack::metric::MahalanobisDistance` + - `mlpack::metric::LMetric` (for arbitrary L-metrics) + - `mlpack::metric::IPMetric` (requires a [KernelType](kerneltype.md) parameter) diff --git a/doc/developer/timer.md b/doc/developer/timer.md new file mode 100644 index 0000000000..a8751bad33 --- /dev/null +++ b/doc/developer/timer.md @@ -0,0 +1,68 @@ +# mlpack Timers + +mlpack provides a simple timer interface for the timing of machine learning +methods. The results of any timers used during the program are displayed at +output by any command-line binding, when `--verbose` is given: + +```sh +$ mlpack_knn -r dataset.csv -n neighbors_out.csv -d distances_out.csv -k 5 -v +<...> +[INFO ] Program timers: +[INFO ] computing_neighbors: 0.010650s +[INFO ] loading_data: 0.002567s +[INFO ] saving_data: 0.001115s +[INFO ] total_time: 0.149816s +[INFO ] tree_building: 0.000534s +``` + +## Timer API + +In C++, the `mlpack::Timers` class can be used to add timers to a program. The +`mlpack::Timers` class provides three simple methods: + +```c++ +void Timer::Start(const char* name); +void Timer::Stop(const char* name); +timeval Timer::Get(const char* name); +``` + +Every binding is called with an `mlpack::Timers&`, which can be used in the body +of that binding. For the sake of this discussion, let us call that object +`timers`. + +Each timer is given a name, and is referenced by that name. You can call +`timers.Start()` and `timers.Stop()` multiple times for a particular timer name, +and the result will be the sum of the runs of the timer. Note that +`timers.Stop()` must be called before `timers.Start()` is called again, +otherwise a `std::runtime_error` exception will be thrown. + +A `"total_time"` timer is run automatically for each mlpack binding. + +## Timer Example + +Below is a very simple example of timer usage in code. + +```c++ +#include +#include +#define BINDING_TYPE BINDING_TYPE_CLI +#include + +using namespace mlpack; + +void BINDING_FUNCTION(util::Params& params, util::Timers& timers) +{ + // Start a timer. + timers.Start("some_timer"); + + // Do some things. + DoSomeStuff(); + + // Stop the timer. + timers.Stop("some_timer"); +} +@endcode + +If the `verbose` flag was given to this binding, then a command-line binding +would print the time that `"some_timer"` ran for at the end of the program's +output. diff --git a/doc/policies/trees.hpp b/doc/developer/trees.md similarity index 61% rename from doc/policies/trees.hpp rename to doc/developer/trees.md index 36a03e634a..807c61ed06 100644 --- a/doc/policies/trees.hpp +++ b/doc/developer/trees.md @@ -1,6 +1,4 @@ -/*! @page trees The TreeType policy in mlpack - -@section treeintro Introduction +# The TreeType policy in mlpack Trees are an important data structure in mlpack and are used in a number of the machine learning algorithms that mlpack implements. Often, the use of trees can @@ -8,58 +6,42 @@ allow significant acceleration of an algorithm; this is generally done by pruning away large parts of the tree during computation. Most mlpack algorithms that use trees are not tied to a specific tree but -instead allow the user to choose a tree via the \c TreeType template parameter. -Any tree passed as a \c TreeType template parameter will need to implement a +instead allow the user to choose a tree via the `TreeType` template parameter. +Any tree passed as a `TreeType` template parameter will need to implement a certain set of functions. In addition, a tree may optionally specify some -traits about itself with the \c TreeTraits trait class. +traits about itself with the `TreeTraits` trait class. This document aims to clarify the abstractions underlying mlpack trees, list and -describe the required functionality of the \c TreeType policy, and point users -towards existing types of trees. A table of contents is below: - - - \ref treeintro - - \ref whatistree - - \ref treetype_template_params - - \ref treetype_api - - \ref treetype_rigorous - - \ref treetype_rigorous_template - - \ref treetype_rigorous_constructor - - \ref treetype_rigorous_basic - - \ref treetype_rigorous_complex - - \ref treetype_rigorous_serialization - - \ref treetype_traits - - \ref treetype_more +describe the required functionality of the `TreeType` policy, and point users +towards existing types of trees. Although this document is long, there may still be errors and unclear areas. If you are having trouble understanding anything, please get in touch on Github or on the mailing list and someone will help you (and possibly update the documentation afterwards). -@section whatistree What is a tree? +## What is a tree? In mlpack, we assume that we have some sort of data matrix, which might be -sparse or dense (that is, it could be of type \c arma::mat or \c arma::sp_mat, +sparse or dense (that is, it could be of type `arma::mat` or `arma::sp_mat`, or any variant that implements the Armadillo API). This data matrix corresponds to a collection of points in some space (usually a Euclidean space). A tree is a way of organizing this data matrix in a hierarchical manner---so, points that are nearby should lie in similar nodes. -We can rigorously define what a tree is, using the definition of **space tree** +We can rigorously define what a tree is, using the definition of *space tree* introduced in the following paper: -@code -@quote +```c++ R.R. Curtin, W.B. March, P. Ram, D.V. Anderson, A.G. Gray, and C.L. Isbell Jr., "Tree-independent dual-tree algorithms," in Proceedings of the 30th International Conference on Machine Learning (ICML '13), pp. 1435--1443, 2013. -@endquote -@endcode +``` The definition is: -A **space tree** on a dataset \f$ S \in \mathcal{R}^{N \times d} \f$ is an -undirected, connected, acyclic, rooted simple graph with the following -properties: +A *space tree* on a dataset `S` in `R^(N x d)` is an undirected, connected, +acyclic, rooted simple graph with the following properties: - Each node (or vertex) holds a number of points (possibly zero) and is connected to one parent node and a number of child nodes (possibly zero). @@ -67,17 +49,16 @@ connected to one parent node and a number of child nodes (possibly zero). - There is one node in every space tree with no parent; this is the root node of the tree. - - Each point in \f$S\f$ is contained in at least one node. + - Each point in `S` is contained in at least one node. - - Each node corresponds to some subset of \f$\mathcal{R}^d\f$ that contains -each point in the node and also the subsets that correspond to each child of the -node. + - Each node corresponds to some subset of `R^d` that contains each point in the + node and also the subsets that correspond to each child of the node. This is really a quite straightforward definition: a tree is hierarchical, and each node corresponds to some region of the input space. Each node may have some number of children, and may hold some number of points. However, there is -an important terminology distinction to make: the term **points held by a node** -has a different meaning than the term **descendant points held by a node**. The +an important terminology distinction to make: the term *points held by a node* +has a different meaning than the term *descendant points held by a node*. The points held in a node are just that---points held only in the node. The descendant points of a node are the combination of the points held in a node with the points held in the node's children and the points held in the node's @@ -89,37 +70,37 @@ Now, it's also important to note that a point does not *need* to hold any children, and that a node *can* hold the same points as its children (or its parent). Some types of trees do this. For instance, each node in the cover tree holds only one point, and may have a child that holds the same point. As -another example, the \f$kd\f$-tree holds its points only in the leaves (at the +another example, the `kd`-tree holds its points only in the leaves (at the bottom of the tree). More information on space trees can be found in either the "Tree-independent dual-tree algorithms" paper or any of the related literature. So there is a huge amount of possible variety in the types of trees that can fall into the class of *space trees*. Therefore, it's important to treat them -abstractly, and the \c TreeType policy allows us to do just that. All we need +abstractly, and the `TreeType` policy allows us to do just that. All we need to remember is that a node in a tree can be represented as the combination of some points held in the node, some child nodes, and some geometric structure that represents the space that all of the descendant points fall into (this is a restatement of the fourth part of the definition). -@section treetype_template_params Template parameters required by the TreeType policy +## Template parameters required by the TreeType policy Most everything in mlpack is decomposed into a series of configurable template parameters, and trees are no exception. In order to ease usage of high-level mlpack algorithms, each \c TreeType itself must be a template class taking three parameters: - - \c MetricType -- the underlying metric that the tree will be built on (see -\ref metrics "the MetricType policy documentation") - - \c StatisticType -- holds any auxiliary information that individual + - `MetricType` -- the underlying metric that the tree will be built on (see +[the MetricType policy documentation](metrictype.md)) + - `StatisticType` -- holds any auxiliary information that individual algorithms may need - - \c MatType -- the type of the matrix used to represent the data + - `MatType` -- the type of the matrix used to represent the data The reason that these three template parameters are necessary is so that each -\c TreeType can be used as a template template parameter, which can radically +`TreeType` can be used as a template template parameter, which can radically simplify the required syntax for instantiating mlpack algorithms. By using template template parameters, a user needs only to write -@code +```c++ // The RangeSearch class takes a MetricType and a TreeType template parameter. // This code instantiates RangeSearch with the ManhattanDistance and a @@ -128,20 +109,20 @@ template template parameters, a user needs only to write // This example ignores the constructor parameters, for the sake of simplicity. RangeSearch rs(...); -@endcode +``` as opposed to the far more complicated alternative, where the user must specify the values of each template parameter of the tree type: -@code +```c++ // This is a much worse alternative, where the user must specify the template // arguments of their tree. RangeSearch> rs(...); -@endcode +``` Unfortunately, the price to pay for this user convenience is that *every* -\c TreeType must have three template parameters, and they must be in exactly +`TreeType` must have three template parameters, and they must be in exactly that order. Fortunately, there is an additional benefit: we are guaranteed that the tree is built using the same metric as the method (that is, a user can't specify different metric types to the algorithm and to the tree, which they can @@ -149,24 +130,24 @@ without template template parameters). There are two important notes about this: - - Not every possible input of MetricType, StatisticType, and/or MatType -necessarily need to be valid or work correctly for each type of tree. For -instance, the QuadTree is limited to Euclidean metrics and will not work -otherwise. Either compile-time static checks or detailed documentation can help -keep users from using invalid combinations of template arguments. + - Not every possible input of `MetricType`, `StatisticType`, and/or `MatType` + necessarily need to be valid or work correctly for each type of tree. For + instance, the `QuadTree` is limited to Euclidean metrics and will not work + otherwise. Either compile-time static checks or detailed documentation can + help keep users from using invalid combinations of template arguments. - Some types of trees have more template parameters than just these three. One -example is the generalized binary space tree, where the bounding shape of each -node is easily made into a fourth template parameter (the \c BinarySpaceTree -class calls this the \c BoundType parameter), and the procedure used to split a -node is easily made into a fifth template parameter (the \c BinarySpaceTree -class calls this the \c SplitType parameter). However, the syntax of template -template parameters *requires* that the class only has the correct number of -template parameters---no more, no less. Fortunately, C++11 allows template -typedefs, which can be used to provide partial specialization of template -classes: + example is the generalized binary space tree, where the bounding shape of + each node is easily made into a fourth template parameter (the + `BinarySpaceTree` class calls this the `BoundType` parameter), and the + procedure used to split a node is easily made into a fifth template parameter + (the `BinarySpaceTree` class calls this the `SplitType` parameter). However, + the syntax of template template parameters *requires* that the class only has + the correct number of template parameters---no more, no less. Fortunately, + C++11 allows template typedefs, which can be used to provide partial + specialization of template classes: -@code +```c++ // This is the definition of the BinarySpaceTree class, which has five template // parameters. template MeanSplit>; -@endcode +``` -Now, the \c MeanSplitKDTree class has only three template parameters and can be -used as a \c TreeType policy class in various mlpack algorithms. Many types of +Now, the `MeanSplitKDTree` class has only three template parameters and can be +used as a `TreeType` policy class in various mlpack algorithms. Many types of trees in mlpack have more than three template parameters and rely on template -typedefs to provide simplified \c TreeType interfaces. +typedefs to provide simplified `TreeType` interfaces. -@section treetype_api The TreeType API +## The TreeType API As a result of the definition of *space tree* in the previous section, a simplified API presents itself quite easily. However, more complex functionality is often necessary in mlpack, so this leads to more functions -being necessary for a class to satisfy the \c TreeType policy. Combining this +being necessary for a class to satisfy the `TreeType` policy. Combining this with the template parameters required for trees given in the previous section -gives us the complete API required for a class implementing the \c TreeType -policy. Below is the minimal set of functions required with minor -documentation for each function. (More extensive documentation and explanation -is given afterwards.) +gives us the complete API required for a class implementing the `TreeType` +policy. Below is the minimal set of functions required with minor documentation +for each function. (More extensive documentation and explanation is given +afterwards.) -@code +```c++ // The three template parameters will be supplied by the user, and are detailed // in the previous section. template double Evaluate(const VecTypeA& a, const VecTypeB& b); -@endcode +``` -Note that this method is not necessarily static, so a \c MetricType object -should be held internally and its \c Evaluate() method should be called whenever -the distance between two points is required. **It is generally a bad idea to -hardcode any distance calculation in your tree.** This will make the tree -unable to generalize to arbitrary metrics. If your tree must depend on certain +Note that this method is not necessarily static, so a `MetricType` object should +be held internally and its `Evaluate()` method should be called whenever the +distance between two points is required. *It is generally a bad idea to +hardcode any distance calculation in your tree.* This will make the tree unable +to generalize to arbitrary metrics. If your tree must depend on certain assumptions holding about the metric (i.e. the metric is a Euclidean metric), then make that clear in the documentation of the tree, so users do not try to use the tree with an inappropriate metric. -The second template parameter, \c StatisticType, is for auxiliary information +The second template parameter, `StatisticType`, is for auxiliary information that is required by certain algorithms. For instance, consider an algorithm which repeatedly uses the variance of the descendant points of a node. It might -be tempting to add a \c Variance() method to the required \c TreeType API, but +be tempting to add a `Variance()` method to the required `TreeType` API, but this quickly leads to code bloat (after all, the API already has quite enough -functions as it is). Instead, it is better to create a \c StatisticType class -which provides the \c Variance() method, and then call \c Stat().Variance() when +functions as it is). Instead, it is better to create a `StatisticType` class +which provides the `Variance()` method, and then call `Stat().Variance()` when the variance is required. This also holds true for cached data members. -Each node should have its own instance of a \c StatisticType class. The -\c StatisticType must provide the following constructors: +Each node should have its own instance of a `StatisticType` class. The +`StatisticType` must provide the following constructors: -@code +```c++ // Default constructor required by the StatisticType policy. StatisticType(); // This constructor is required by the StatisticType policy. template StatisticType(TreeType& node); -@endcode +``` -This constructor should be called with \c (*this) after the node is constructed +This constructor should be called with `(*this)` after the node is constructed (usually, this ends up being the last line in the constructor of a node). -The last template parameter is the \c MatType parameter. This is generally -\c arma::mat or \c arma::sp_mat, but could be any Armadillo type, including -matrices that hold data points of different precisions (such as \c float or even -\c int). It generally suffices to write \c MatType assuming that \c arma::mat +The last template parameter is the `MatType` parameter. This is generally +`arma::mat` or `arma::sp_mat`, but could be any Armadillo type, including +matrices that hold data points of different precisions (such as `float` or even +`int`). It generally suffices to write \c MatType assuming that `arma::mat` will be used, since the vast majority of the time this will be what is used. -@subsection treetype_rigorous_constructor Constructors and destructors +### Constructors and destructors -The \c TreeType API requires at least three constructors. Technically, it does +The `TreeType` API requires at least three constructors. Technically, it does not *require* a destructor, but almost certainly your tree class will be doing some memory management internally and should have one (though not always). The first two constructors are variations of the same idea: -@code +```c++ // This batch constructor does not modify the dataset, and builds the entire // tree using a default-constructed MetricType. ExampleTree(const MatType& data); @@ -468,19 +443,19 @@ ExampleTree(const MatType& data); // This batch constructor does not modify the dataset, and builds the entire // tree using the given MetricType. ExampleTree(const MatType& data, MetricType& metric); -@endcode +``` All that is required here is that a constructor is available that takes a dataset and optionally an instantiated metric. If no metric is provided, then -it should be assumed that the \c MetricType class has a default constructor and +it should be assumed that the `MetricType` class has a default constructor and a default-constructed metric should be used. The constructor *must* return a valid, fully-constructed, ready-to-use tree that satisfies the definition -of *space tree* that was \ref whatistree "given earlier". +of *space tree* that was given earlier in the document. -The third constructor requires the tree to be initializable from a \c -cereal archive: +The third constructor requires the tree to be initializable from a `cereal` +archive: -@code +```c++ // Initialize the tree from a given cereal archive. SFINAE (the // second argument) is necessary to ensure that the archive is loading, not // saving. @@ -488,81 +463,80 @@ template ExampleTree( Archive& ar, const typename std::enable_if_c::type* = 0); -@endcode +``` This has implications on how the tree must be stored. In this case, the dataset -is *not yet loaded* and therefore the tree **may be required to have -ownership of the data matrix**. This means that realistically the most +is *not yet loaded* and therefore the tree ***may be required to have +ownership of the data matrix***. This means that realistically the most reasonable way to represent the data matrix internally in a tree class is not with a reference but instead with a pointer. If this is true, then a destructor will be required: -@code +```c++ // Release any resources held by the tree. ~ExampleTree(); -@endcode +``` and, if the data matrix is represented internally with a pointer, this destructor will need to release the memory for the data matrix (in the case that -the tree was created via \c cereal ). +the tree was created via `cereal`). Note that these constructors are not necessarily the only constructors that a -\c TreeType implementation can provide. One important example of when more +`TreeType` implementation can provide. One important example of when more constructors are useful is when the tree rearranges points internally; this might be desired for the sake of speed or memory optimization. But to do this with the required constructors would necessarily incur a copy of the data -matrix, because the user will pass a \c "const MatType&". One alternate -solution is to provide a constructor which takes an rvalue reference to a -\c MatType: +matrix, because the user will pass a `const MatType&`. One alternate solution +is to provide a constructor which takes an rvalue reference to a `MatType`: -@code +```c++ template ExampleTree(MatType&& data); -@endcode +``` (and another overload that takes an instantiated metric), and then the user can -use \c std::move() to build the tree without copying the data matrix, although +use `std::move()` to build the tree without copying the data matrix, although the data matrix will be modified: -@code +```c++ ExampleTree exTree(std::move(dataset)); -@endcode +``` It is, of course, possible to add even more constructors if desired. -@subsection treetype_rigorous_basic Basic tree functionality +### Basic tree functionality -The basic functionality of a class implementing the \c TreeType API is quite +The basic functionality of a class implementing the `TreeType` API is quite straightforward and intuitive. -@code +```c++ // Get the dataset that the tree is built on. const MatType& Dataset(); -@endcode +``` -This should return a \c const reference to the dataset the tree is built on. -The fact that this function is required essentially means that each node in the -tree must store a pointer to the dataset (this is not the only option, but it is -the most obvious option). +This should return a `const` reference to the dataset the tree is built on. The +fact that this function is required essentially means that each node in the tree +must store a pointer to the dataset (this is not the only option, but it is the +most obvious option). -@code +```c++ // Get the metric that the tree is built with. MetricType& Metric(); -@endcode +``` Each node must also store an instantiated metric or a pointer to one (note that -this is required even for metrics that have no state and have a \c static \c -Evaluate() function). +this is required even for metrics that have no state and have a `static` +`Evaluate()` function). -@code +```c++ // Get/modify the StatisticType for this node. StatisticType& Stat(); -@endcode +``` -As discussed earlier, each node must hold a \c StatisticType; this is accessible -through the \c Stat() function. +As discussed earlier, each node must hold a `StatisticType`; this is accessible +through the `Stat()` function. -@code +```c++ // Return the parent of the node, or NULL if this is the root. ExampleTree* Parent(); @@ -585,63 +559,63 @@ ExampleTree& DescendantNode(const size_t i); size_t NumDescendants(); // Return the index of the i'th descendant point of this node. size_t Descendant(const size_t i); -@endcode +``` These functions are all fairly self-explanatory. Most algorithms will use the -\c Parent(), \c Children(), \c NumChildren(), \c Point(), and \c NumPoints() +`Parent()`, `Children()`, `NumChildren()`, `Point()`, and `NumPoints()` functions, so care should be taken when implementing those functions to ensure -they will be efficient. Note that \c Point() and \c Descendant() should return +they will be efficient. Note that `Point()` and `Descendant()` should return indices of points, so the actual points can be accessed by calling -\c "Dataset().col(Point(i))" for some index \c i (or something similar). +`Dataset().col(Point(i))` for some index `i` (or something similar). -An important note about the \c Descendant() function is that each descendant +An important note about the `Descendant()` function is that each descendant point should be unique. So if a node holds the point with index 6 and it has -one child that holds the points with indices 6 and 7, then \c NumDescendants() +one child that holds the points with indices 6 and 7, then `NumDescendants()` should return 2, not 3. The ordering in which the descendants are returned can -be arbitrary; so, \c Descendant(0) can return 6 \b or 7, and \c Descendant(1) +be arbitrary; so, `Descendant(0)` can return 6 *or* 7, and `Descendant(1)` should return the other index. -@code +```c++ // Store the center of the bounding region of the node in the given vector. void Center(arma::vec& center); -@endcode +``` -The last function, \c Center(), should calculate the center of the bounding -shape and store it in the given vector. So, for instance, if the tree is a ball -tree, then the center is simply the center of the ball. Algorithm writers would -be wise to try and avoid the use of \c Center() if possible, since it will +The last function, `Center()`, should calculate the center of the bounding shape +and store it in the given vector. So, for instance, if the tree is a ball tree, +then the center is simply the center of the ball. Algorithm writers would be +wise to try and avoid the use of `Center()` if possible, since it will necessarily cost a copy of a vector. -@subsection treetype_rigorous_complex Complex tree functionality and bounds +### Complex tree functionality and bounds A node in a tree should also be able to calculate various distance-related bounds; these are particularly useful in tree-based algorithms. Note that any of these bounds does not necessarily need to be maximally tight; generally it is more important that each bound can be easily calculated. -Details on each bounding function that the \c TreeType API requires are given +Details on each bounding function that the `TreeType` API requires are given below. -@code +```c++ // Return the distance between the center of this node and the center of // its parent. double ParentDistance(); -@endcode +``` Remember that each node corresponds to some region in the space that the dataset lies in. For most tree types this shape is often something geometrically simple: a ball, a cone, a hyperrectangle, a slice, or something similar. The -\c ParentDistance() function should return the distance between the center of +`ParentDistance()` function should return the distance between the center of this node's region and the center of the parent node's region. In practice this bound is often used in dual-tree (or single-tree) algorithms to -place an easy \c MinDistance() (or \c MaxDistance() ) bound for a child node; -the parent's \c MinDistance() (or \c MaxDistance() ) function is called and then -adjusted with \c ParentDistance() to provide a possibly loose but efficient -bound on what the result of \c MinDistance() (or \c MaxDistance() ) would be -with the child. +place an easy `MinDistance()` (or `MaxDistance()`) bound for a child node; the +parent's `MinDistance()` (or `MaxDistance()`) function is called and then +adjusted with `ParentDistance()` to provide a possibly loose but efficient bound +on what the result of `MinDistance()` (or `MaxDistance()`) would be with the +child. -@code +```c++ // Return an upper bound on the furthest possible distance between the // center of the node and any point held in the node. double FurthestPointDistance(); @@ -649,25 +623,25 @@ double FurthestPointDistance(); // Return an upper bound on the furthest possible distance between the // center of the node and any descendant point of the node. double FurthestDescendantDistance(); -@endcode +``` It is often very useful to be able to bound the radius of a node, which is -effectively what \c FurthestDescendantDistance() does. Often it is easiest to +effectively what `FurthestDescendantDistance()` does. Often it is easiest to simply calculate and cache the furthest descendant distance at tree construction time. Some trees, such as the cover tree, are able to give guarantees that the points held in the node will necessarily be closer than the descendant points; -therefore, the \c FurthestPointDistance() function is also useful. +therefore, the `FurthestPointDistance()` function is also useful. -It is permissible to simply have \c FurthestPointDistance() return the result of -\c FurthestDescendantDistance(), and that will still be a valid bound, but -depending on the type of tree it may be possible to have \c -FurthestPointDistance() return a tighter bound. +It is permissible to simply have `FurthestPointDistance()` return the result of +`FurthestDescendantDistance()`, and that will still be a valid bound, but +depending on the type of tree it may be possible to have +`FurthestPointDistance()` return a tighter bound. -@code +```c++ // Return a lower bound on the minimum distance between the center and any // edge of the node's bounding shape. double MinimumBoundDistance(); -@endcode +``` This is, admittedly, a somewhat complex and weird quantity. It is one of the less important bounding functions, so it is valid to simply return 0... @@ -675,14 +649,14 @@ less important bounding functions, so it is valid to simply return 0... The bound is a bound on the minimum distance between the center of the node and any edge of the shape that bounds all of the descendants of the node. So, if the bounding shape is a ball (as in a ball tree or a cover tree), then -\c MinimumBoundDistance() should just return the radius of the ball. If the +`MinimumBoundDistance()` should just return the radius of the ball. If the bounding shape is a hypercube (as in a generalized octree), then -\c MinimumBoundDistance() should return the side length divided by two. If the +`MinimumBoundDistance()` should return the side length divided by two. If the bounding shape is a hyperrectangle (as in a kd-tree or a spill tree), then -\c MinimumBoundDistance() should return half the side length of the +`MinimumBoundDistance()` should return half the side length of the hyperrectangle's smallest side. -@code +```c++ // Return a lower bound on the minimum distance between the given point and // the node. template @@ -707,7 +681,7 @@ math::Range RangeDistance(VecType& point); // Return the combined results of MinDistance() and MaxDistance(). math::Range RangeDistance(ExampleTree& otherNode); -@endcode +``` These six functions are almost without a doubt the most important functionality of a tree. Therefore, it is preferable that these methods be implemented as @@ -718,14 +692,14 @@ work, and tighter bounds mean that more pruning is possible. Of these six functions, there are only really two bounds that are desired here: the *minimum distance* between a node and an object, and the *maximum distance* -between a node and an object. The object may be either a vector (usually \c -arma::vec ) or another tree node. +between a node and an object. The object may be either a vector (usually +`arma::vec`) or another tree node. Consider the first case, where the object is a vector. The result of -\c MinDistance() needs to be less than or equal to the true minimum distance, +`MinDistance()` needs to be less than or equal to the true minimum distance, which could be calculated as below: -@code +```c++ // We assume that we have a vector 'vec', and a tree node 'node'. double trueMinDist = DBL_MAX; for (size_t i = 0; i < node.NumDescendants(); ++i) @@ -737,45 +711,31 @@ for (size_t i = 0; i < node.NumDescendants(); ++i) } // At the end of the loop, trueMinDist will hold the true minimum distance // between 'vec' and any descendant point of 'node'. -@endcode +``` Often the bounding shape of a node will allow a quick calculation that will make a reasonable bound. For instance, if the node's bounding shape is a ball with -radius \c r and center \c ctr, the calculation is simply -\c "(node.Metric().Evaluate(vec, ctr) - r)". Usually a good \c MinDistance() or -\c MaxDistance() function will make only one call to the \c Evaluate() function -of the metric. +radius `r` and center `ctr`, the calculation is simply +`(node.Metric().Evaluate(vec, ctr) - r)`. Usually a good `MinDistance()` or +`MaxDistance()` function will make only one call to the `Evaluate()` function of +the metric. -The \c RangeDistance() function allows a way for both bounds to be calculated at -once. It is possible to implement this as a call to \c MinDistance() followed -by a call to \c MaxDistance(), but this may incur more metric \c Evaluate() -calls than necessary. Often calculating both bounds at once can be more -efficient and can be done with fewer \c Evaluate() calls than calling both -\c MinDistance() and \c MaxDistance(). +The `RangeDistance()` function allows a way for both bounds to be calculated at +once. It is possible to implement this as a call to `MinDistance()` followed by +a call to `MaxDistance()`, but this may incur more metric `Evaluate()` calls +than necessary. Often calculating both bounds at once can be more efficient and +can be done with fewer `Evaluate()` calls than calling both `MinDistance()` and +`MaxDistance()`. -@subsection treetype_rigorous_serialization Serialization +### Serialization -The last two public functions that the \c TreeType API requires are related to -serialization and printing. +The last functions that the `TreeType` API requires are for serialization. -@code -// Return a string representation of the tree. -std::string ToString() const; -@endcode - -There are few restrictions on the precise way that the \c ToString() function -should operate, but generally it should behave similarly to the \c ToString() -function in other mlpack methods. Generally, a user will call \c ToString() -when they want to inspect the object and see what it looks like. For a tree, -printing the entire tree may be way more information than the user was -expecting, so it may be a better option to print either only the node itself or -the node plus one or two levels of children. - -@code +```c++ // Serialize the tree (load from the given archive / save to the given // archive, depending on its type). template -void serialize(Archive& ar); +void serialize(Archive& ar, const unsigned int version); protected: // A default constructor; only meant to be used by cereal. This @@ -785,38 +745,37 @@ ExampleTree(); // Friend access must be given for the default constructor. friend class cereal::access; -@endcode +``` On the other hand, the specifics of the functionality required for the -\c Serialize() function are somewhat more difficult. The \c Serialize() -function will be called either when a tree is being saved to disk or loaded from -disk. The \c cereal documentation is fairly comprehensive. +`serialize()` function are somewhat more difficult. The `serialize()` function +will be called either when a tree is being saved to disk or loaded from disk. +The `cereal` documentation is fairly comprehensive. -An important note is that it is very difficult to use references with -\c cereal, because \c serialize() may be called at any time during -the object's lifetime, and references cannot be re-seated. In general this will -require the use of pointers, which then require manual memory management. -Therefore, be careful that \c serialize() (and the tree's destructor) properly -handle memory management! +An important note is that it is very difficult to use references with `cereal`, +because `serialize()` may be called at any time during the object's lifetime, +and references cannot be re-seated. In general this will require the use of +pointers, which then require manual memory management. Therefore, be careful +that `serialize()` (and the tree's destructor) properly handle memory +management! -@section treetype_traits The TreeTraits trait class +## The TreeTraits trait class Some tree-based algorithms can specialize if the tree fulfills certain conditions. For instance, if the regions represented by two sibling nodes cannot overlap, an algorithm may be able to perform a simpler computation. -Based on this reasoning, the \c TreeTraits trait class (much like the -mlpack::kernel::KernelTraits class) exists in order to allow a tree to specify -(via a \c const \c static \c bool) when these types of conditions are -satisfied. **Note that a TreeTraits class is not required,** but may be -helpful. +Based on this reasoning, the `TreeTraits` trait class (much like the +`mlpack::kernel::KernelTraits` class) exists in order to allow a tree to specify +(via a `const static bool`) when these types of conditions are satisfied. +***Note that a TreeTraits class is not required***, but may be helpful. -The \c TreeTraits trait class is a template class that takes a \c TreeType as a -parameter, and exposes \c const \c static \c bool values that depend on the -tree. Setting these values is achieved by specialization. The code below shows -the default \c TreeTraits values (these are the values that will be used if no -specialization is provided for a given \c TreeType). +The `TreeTraits` trait class is a template class that takes a `TreeType` as a +parameter, and exposes `const static bool` values that depend on the tree. +Setting these values is achieved by specialization. The code below shows the +default `TreeTraits` values (these are the values that will be used if no +specialization is provided for a given `TreeType`). -@code +```c++ template class TreeTraits { @@ -838,14 +797,14 @@ class TreeTraits // This is true if the tree always has only two children. static const bool BinaryTree = false; }; -@endcode +``` -An example specialization for the \ref mlpack::tree::KDTree class is given -below. Note that \ref mlpack::tree::KDTree is itself a template class (like -every class satisfying the \c TreeType policy), so we are specializing to a -template parameter. +An example specialization for the `mlpack::tree::KDTree` class is given below. +Note that `mlpack::tree::KDTree` is itself a template class (like every class +satisfying the `TreeType` policy), so we are specializing to a template +parameter. -@code +```c++ template @@ -868,29 +827,28 @@ class TreeTraits> // The tree is always binary. static const bool BinaryTree = true; }; -@endcode +``` Currently, the traits available are each of the five detailed above. For more -information, see the \ref mlpack::tree::TreeTraits documentation. +information, see the `mlpack::tree::TreeTraits` source code for more +documentation. -@section treetype_more A list of trees in mlpack and more information +## A list of trees in mlpack and more information mlpack contains several ready-to-use implementations of trees that satisfy the TreeType policy API: - - mlpack::tree::KDTree - - mlpack::tree::MeanSplitKDTree - - mlpack::tree::BallTree - - mlpack::tree::MeanSplitBallTree - - mlpack::tree::RTree - - mlpack::tree::RStarTree - - mlpack::tree::StandardCoverTree + - `mlpack::tree::KDTree` + - `mlpack::tree::MeanSplitKDTree` + - `mlpack::tree::BallTree` + - `mlpack::tree::MeanSplitBallTree` + - `mlpack::tree::RTree` + - `mlpack::tree::RStarTree` + - `mlpack::tree::StandardCoverTree` Often, these are template typedefs of more flexible tree classes: - - mlpack::tree::BinarySpaceTree -- binary trees, such as the KD-tree and ball + - `mlpack::tree::BinarySpaceTree` -- binary trees, such as the KD-tree and ball tree - - mlpack::tree::RectangleTree -- the R tree and variants - - mlpack::tree::CoverTree -- the cover tree and variants - -*/ + - `mlpack::tree::RectangleTree` -- the R tree and variants + - `mlpack::tree::CoverTree` -- the cover tree and variants diff --git a/doc/developer/version.md b/doc/developer/version.md new file mode 100644 index 0000000000..c303f2a624 --- /dev/null +++ b/doc/developer/version.md @@ -0,0 +1,25 @@ +# mlpack versions in code + +mlpack provides a couple of convenience macros and functions to get the version +of mlpack. More information (and straightforward code) can be found in +`src/mlpack/core/util/version.hpp`. + +The following three macros provide major, minor, and patch versions of mlpack +(i.e. for `mlpack-x.y.z`, `x` is the major version, `y` is the minor version, +and `z` is the patch version): + +```c++ +MLPACK_VERSION_MAJOR +MLPACK_VERSION_MINOR +MLPACK_VERSION_PATCH +``` + +In addition, the function `mlpack::util::GetVersion()` returns the mlpack +version as a string (for instance, `"mlpack 1.0.8"`). + +## mlpack command-line program versions + +Each mlpack command-line program supports the `--version` (or `-V`) option, +which will print the version of mlpack used. If the version is not an official +release but instead from git, the version will be `mlpack git` (and will have a +git revision SHA appended to `git`). diff --git a/doc/doxygen/extra-stylesheet.css b/doc/doxygen/extra-stylesheet.css deleted file mode 100644 index a9f808272f..0000000000 --- a/doc/doxygen/extra-stylesheet.css +++ /dev/null @@ -1,7 +0,0 @@ -/* Additional CSS styles for the html output */ - -/* Fix the size of inline formulas */ -img.formulaInl { - vertical-align: middle; - height: 15pt; -} diff --git a/doc/doxygen/footer.html b/doc/doxygen/footer.html deleted file mode 100644 index 730f025aaa..0000000000 --- a/doc/doxygen/footer.html +++ /dev/null @@ -1,16 +0,0 @@ - -
-Generated by   -doxygen - $doxygenversion -
- - - diff --git a/doc/doxygen/stylesheet.css b/doc/doxygen/stylesheet.css deleted file mode 100644 index 5f92436b98..0000000000 --- a/doc/doxygen/stylesheet.css +++ /dev/null @@ -1,888 +0,0 @@ -/* The standard CSS for doxygen */ - -body, table, div, p, dl { - font-family: Lucida Grande, Verdana, Geneva, Arial, sans-serif; - font-size: 12px; -} - -/* @group Heading Levels */ - -h1 { - font-size: 150%; - color: #ffffff; -} - -.title { - font-size: 150%; - font-weight: bold; - margin: 10px 2px; - color: #ffffff; -} - -h2 { - font-size: 120%; - color: #ffffff; -} - -h3 { - font-size: 100%; - color: #ffffff; -} - -dt { - font-weight: bold; -} - -div.multicol { - -moz-column-gap: 1em; - -webkit-column-gap: 1em; - -moz-column-count: 3; - -webkit-column-count: 3; -} - -p.startli, p.startdd, p.starttd { - margin-top: 2px; -} - -p.endli { - margin-bottom: 0px; -} - -p.enddd { - margin-bottom: 4px; -} - -p.endtd { - margin-bottom: 2px; -} - -/* @end */ - -caption { - font-weight: bold; -} - -span.legend { - font-size: 70%; - text-align: center; -} - -h3.version { - font-size: 90%; - text-align: center; -} - -div.qindex, div.navtab{ - background-color: #000000; - border: 1px solid #333333; - text-align: center; - margin: 2px; - padding: 2px; -} - -div.qindex, div.navpath { - width: 100%; - line-height: 140%; -} - -div.navtab { - margin-right: 15px; -} - -/* @group Link Styling */ - -a { - color: #BB2222; - font-weight: normal; - text-decoration: none; -} - -.contents a:visited { - color: #BB2222; -} - -a:hover { - text-decoration: underline; -} - -a.qindex { - font-weight: bold; -} - -a.qindexHL { - font-weight: bold; - background-color: #9CAFD4; - color: #ffffff; - border: 1px double #869DCA; -} - -.contents a.qindexHL:visited { - color: #ffffff; -} - -a.el { - font-weight: bold; -} - -a.elRef { - -} - -a.code { - color: #BB2222; -} - -a.codeRef { - color: #BB2222; -} - -/* @end */ - -dl.el { - margin-left: -1cm; -} - -.fragment { - font-family: monospace, fixed; - font-size: 105%; -} - -pre.fragment { - border: 5px solid #1D1D1D; - background-color: #2D2D2D; - padding: 10px 10px 10px 10px; - page-break-before: avoid; - overflow: auto; - word-wrap: break-word; - font-size: 90%; - margin-left: 1.75em; - margin-right: 1.75em; - margin-top: 1em; - margin-bottom: 1em; - color: #ffffff; -} - -div.ah { - background-color: black; - font-weight: bold; - color: #ffffff; - margin-bottom: 3px; - margin-top: 3px; - padding: 0.2em; - border: solid thin #333; -} - -div.groupHeader { - margin-left: 16px; - margin-top: 12px; - font-weight: bold; -} - -div.groupText { - margin-left: 16px; - font-style: italic; -} - -body { - background: #000000; - color: #808080; - margin: 0; -} - -div.contents { - margin-top: 10px; - margin-left: 10px; - margin-right: 5px; -} - -td.indexkey { - background-color: #000000; - font-weight: bold; - border: 1px solid #333333; - margin: 2px 0px 2px 0; - padding: 2px 10px; -} - -td.indexvalue { - background-color: #000000; - border: 1px solid #333333; - padding: 2px 10px; - margin: 2px 0px; -} - -tr.memlist { - background-color: #EEF1F7; -} - -p.formulaDsp { - text-align: center; -} - -img.formulaDsp { - -} - -img.formulaInl { - vertical-align: middle; -} - -div.center { - text-align: center; - margin-top: 0px; - margin-bottom: 0px; - padding: 0px; -} - -div.center img { - border: 0px; -} - -address.footer { - text-align: right; - padding-right: 12px; -} - -img.footer { - border: 0px; - vertical-align: middle; -} - -/* @group Code Colorization */ - -span.keyword { - color: #FF0000; -} - -span.keywordtype { - color: #FF00FF; -} - -span.keywordflow { - color: #800080; -} - -span.comment { - color: #00FFFF; -} - -span.preprocessor { - color: #808080; -} - -span.stringliteral { - color: #FFFF00; -} - -span.charliteral { -color: #FFFF00; -} - -span.vhdldigit { - color: #FFFF00; -} - -span.vhdlchar { - color: #FFFF00; -} - -span.vhdlkeyword { - color: #FF0000; -} - -span.vhdllogic { - color: #FF0000; -} - -/* @end */ - -/* - .search { -color: #003399; -font-weight: bold; -} - -form.search { -margin-bottom: 0px; -margin-top: 0px; -} - -input.search { -font-size: 75%; -color: #000080; -font-weight: normal; -background-color: #e8eef2; -} - */ - -td.tiny { - font-size: 75%; -} - -.dirtab { - padding: 4px; - border-collapse: collapse; - border: 1px solid #A3B4D7; -} - -th.dirtab { - background: #EBEFF6; - font-weight: bold; -} - -hr { - height: 0px; - border: none; - border-top: 3px solid #BB2222; -} - -hr.footer { - height: 1px; -} - -/* @group Member Descriptions */ - -table.memberdecls { - border-spacing: 0px; - padding: 0px; -} - -.mdescLeft, .mdescRight, -.memItemLeft, .memItemRight, -.memTemplItemLeft, .memTemplItemRight, .memTemplParams { - background-color: #000000; - border: none; - margin: 4px; - padding: 1px 0 0 8px; -} - -.mdescLeft, .mdescRight { - padding: 0px 8px 4px 8px; - color: #555; -} - -.memItemLeft, .memItemRight, .memTemplParams { - border-top: 1px solid #333333; -} - -.memItemLeft, .memTemplItemLeft { - white-space: nowrap; -} - -.memItemRight { - width: 100%; -} - -.memTemplParams { - color: #FFFFFF; - white-space: nowrap; -} - -/* @end */ - -/* @group Member Details */ - -/* Styles for detailed member documentation */ - -.memtemplate { - color: #FFFFFF; - font-weight: bold; - margin-left: 8px; - font-family: Andalo Mono, Courier New, Courier, Lucida Typewrite, fixed; -} - -.memnav { - background-color: #000000; - border: 1px solid #333333; - text-align: center; - margin: 2px; - margin-right: 15px; - padding: 2px; -} - -.mempage { - width: 100%; -} - -.memitem { - padding: 0; - margin-bottom: 10px; - margin-right: 5px; -} - -.memname { - white-space: nowrap; - font-weight: bold; - margin-left: 6px; - font-family: Andale Mono, Courier New, Courier, Lucida Typewriter, fixed; -} - -.memproto { - border-top: 1px solid #808080; - border-left: 1px solid #808080; - border-right: 1px solid #808080; - padding: 6px 0px 6px 0px; - color: #FFFFFF; - font-weight: bold; -} - -.memdoc { - border-bottom: 1px solid #808080; - border-left: 1px solid #808080; - border-right: 1px solid #808080; - border-top: 1px solid #333333; - padding: 2px 5px; -} - -.paramkey { - text-align: right; -} - -.paramtype { - white-space: nowrap; - color: #808080; - font-family: Andale Mono, Courier New, Courier, Lucida Typewriter, fixed; -} - -.paramname { - color: #BB2222; - white-space: nowrap; - font-family: Andale Mono, Courier New, Courier, Lucida Typewriter, fixed; -} - -.paramname em { - font-style: normal; -} - -.params, .retval, .exception, .tparams { - border-spacing: 6px 2px; -} - -.params .paramname, .retval .paramname { - font-weight: bold; - vertical-align: top; -} - -.params .paramtype { - font-style: italic; - vertical-align: top; -} - -.params .paramdir { - font-family: "courier new",courier,monospace; - vertical-align: top; -} - -/* @end */ - -/* @group Directory (tree) */ - -/* for the tree view */ - -.ftvtree { - font-family: sans-serif; - margin: 0px; -} - -/* these are for tree view when used as main index */ - -.directory { - font-size: 9pt; - font-weight: bold; - margin: 5px; -} - -.directory h3 { - margin: 0px; - margin-top: 1em; - font-size: 11pt; -} - -/* - The following two styles can be used to replace the root node title - with an image of your choice. Simply uncomment the next two styles, - specify the name of your image and be sure to set 'height' to the - proper pixel height of your image. - */ - -/* - .directory h3.swap { -height: 61px; -background-repeat: no-repeat; -background-image: url("yourimage.gif"); -} -.directory h3.swap span { -display: none; -} - */ - -.directory > h3 { - margin-top: 0; -} - -.directory p { - margin: 0px; - white-space: nowrap; -} - -.directory div { - display: none; - margin: 0px; -} - -.directory img { - vertical-align: -30%; -} - -/* these are for tree view when not used as main index */ - -.directory-alt { - font-size: 100%; - font-weight: bold; -} - -.directory-alt h3 { - margin: 0px; - margin-top: 1em; - font-size: 11pt; -} - -.directory-alt > h3 { - margin-top: 0; -} - -.directory-alt p { - margin: 0px; - white-space: nowrap; -} - -.directory-alt div { - display: none; - margin: 0px; -} - -.directory-alt img { - vertical-align: -30%; -} - -/* @end */ - -div.dynheader { - margin-top: 8px; -} - -address { - font-style: normal; - color: #2A3D61; -} - -table.doxtable { - border-collapse: collapse; -} - -table.doxtable td, table.doxtable th { - border: 1px solid #2D4068; - padding: 3px 7px 2px; -} - -table.doxtable th { - background-color: #374F7F; - color: #FFFFFF; - font-size: 110%; - padding-bottom: 4px; - padding-top: 5px; - text-align: left; -} - -.tabsearch { - top: 0px; - left: 10px; - height: 36px; - background-image: url('tab_b.png'); - z-index: 101; - overflow: hidden; - font-size: 13px; -} - -.navpath ul { - font-size: 11px; - background: #000000; - color: #8AA0CC; - border-bottom: 1px solid #333333; - overflow: hidden; - margin: 0px; - padding-top: 0.25em; - padding-bottom: 0.25em; - padding-left: 0.5em; - padding-right: 0; - border-left: 1px solid #333333; -} - -.navpath li { - list-style-type: none; - float: left; - padding-right: 0.5em; - color: #364D7C; - border-right: 1px solid #333333; - padding-left: 0.5em; -} - -.navpath li.navelem a { - display: block; - text-decoration: none; - outline: none; -} - -.navpath li.navelem a:hover { - color:#FFFFFF; -} - -.navpath li.footer { - list-style-type: none; - float: right; - padding-left: 10px; - padding-right: 15px; - background-image: none; - background-repeat: no-repeat; - background-position: right; - color: #364D7C; - font-size: 8pt; -} - -div.summary { - float: right; - font-size: 8pt; - padding-right: 5px; - width: 50%; - text-align: right; -} - -div.summary a { - white-space: nowrap; -} - -div.ingroups { - font-size: 8pt; - padding-left: 5px; - width: 50%; - text-align: left; -} - -div.ingroups a { - white-space: nowrap; -} - -div.header { - background-color: #000000; - margin: 0px; - border-bottom: 1px solid #333333; -} - -div.headertitle { - padding: 5px 5px 5px 10px; -} - -dl { - padding: 0 0 0 10px; -} - -dl.note, dl.warning, dl.attention, dl.pre, dl.post, dl.invariant, -dl.deprecated, dl.todo, dl.test, dl.bug { - border-left: 4px solid; - padding: 0 0 0 6px; -} - -dl.note { - border-color: #D0C000; -} - -dl.warning, dl.attention { - border-color: #FF0000; -} - -dl.pre, dl.post, dl.invariant { - border-color: #00D000; -} - -dl.deprecated { - border-color: #505050; -} - -dl.todo { - border-color: #00C0E0; -} - -dl.test { - border-color: #3030E0; -} - -dl.bug { - border-color: #C08050; -} - -#projectlogo { - text-align: center; - vertical-align: bottom; - border-collapse: separate; -} - -#projectlogo img { - border: 0px none; -} - -#projectname { - font: 300% Tahoma, Arial, sans-serif; - margin: 0px; - padding: 2px 0px; -} - -#projectbrief { - font: 120% Tahoma, Arial, sans-serif; - margin: 0px; - padding: 0px; -} - -#projectnumber { - font: 50% Tahoma, Arial,sans-serif; - margin: 0px; - padding: 0px; -} - -#titlearea { - padding: 0px; - margin: 0px; - width: 100%; - border-bottom: 1px solid #808080; -} - -.image { - text-align: center; -} - -.dotgraph { - text-align: center; -} - -.mscgraph { - text-align: center; -} - -.caption { - font-weight: bold; -} - -/** tab list at top of page */ -.tabs, .tabs2, .tabs3 { - background-image: none !important; - background: #000000; - border-left: 1px solid #333333; - border-right: 1px solid #333333; - border-bottom: 1px solid #333333; - min-height: 1.5em; -} - -.tablist li { - background-image: none !important; - background: #000000; - border-right: 1px solid #333333; - height: auto !important; - padding-bottom: 0.25em; - padding-top: 0.25em; - line-height: 1em !important; -} - -.tablist li.current { - background: #BB2222; -} - -.tablist li.current a { - background-image: none !important; - text-shadow: none; - color: #ffffff; -} - -.tablist a { - background-image: none !important; - text-shadow: none; - color: #ffffff; - font-weight: bold; -} - -.tablist li:hover { - background: #333333; -} - -.tablist li.current:hover { - background: #BB2222 !important; -} - -/*** - * For trac-doxygen; these rules won't apply otherwise. - */ -div.tabs span { - background-image: none !important; - background: transparent !important; - height: auto !important; - padding-bottom: 0.25em; - padding-top: 0.25em; - line-height: 1em !important; -} - -div.tabs a { - background-image: none !important; - background: transparent !important; - border-bottom: none !important; - font-size: 100% !important; -} - -div.tabs span { - padding-bottom: 0.25em; - padding-top: 0.25em; - color: #ffffff !important; -} - -div.tabs li:hover { - background: #333333; -} - -div.tabs li.current:hover { - background: #BB2222 !important; -} - -div.tabs li.current { - background: #BB2222 !important; -} - -div.tabs li { - border-right: 1px solid #333333; -} - -div.tabs ul { - display: inline; - font-size: 100%; - padding-top: 0em; -} - -/* I want the menus to display directly below the Trac menu. */ -#content { - padding-top: 0px; - margin-top: 0px; -} - -div.tabs { - margin-bottom: 0px; - background-image: none; -} - -div.nav { - border-bottom: 1px solid #808080; -} - -/*** Fix the weird size of the menus */ -#mainnav { - font-size: 100% !important; -} - -div#main div.nav { - min-height: 1em !important; /* We must have the right height for the menus. */ - border-bottom: 1px solid #333333; /* The plugin was giving a blue border. */ -} diff --git a/doc/examples/sample-ml-app/sample-ml-app/stdafx.h b/doc/examples/sample-ml-app/sample-ml-app/stdafx.h index bf45dc3bf4..ecdac10e4f 100644 Binary files a/doc/examples/sample-ml-app/sample-ml-app/stdafx.h and b/doc/examples/sample-ml-app/sample-ml-app/stdafx.h differ diff --git a/doc/guide/build.hpp b/doc/guide/build.hpp deleted file mode 100644 index 2047feffc3..0000000000 --- a/doc/guide/build.hpp +++ /dev/null @@ -1,341 +0,0 @@ -/*! @page build Building mlpack From Source - -@section build_buildintro Introduction - -This document discusses how to build mlpack from source. These build directions -will work for any Linux-like shell environment (for example Ubuntu, macOS, -FreeBSD etc). However, mlpack is in the repositories of many Linux distributions -and so it may be easier to use the package manager for your system. For example, -on Ubuntu, you can install the mlpack headers and command-line executables (e.g. -mlpack_pca, mlpack_kmeans, etc.) with the following command: - -@code -$ sudo apt-get install libmlpack-dev mlpack-bin -@endcode - -On Fedora or Red Hat(EPEL): - -@code -$ sudo dnf install mlpack-devel mlpack-bin -@endcode - -For installing only the header files for building C++ applications on top of -mlpack, one could use: - -@code -$ sudo apt-get install libmlpack-dev -@endcode - -@note Older Ubuntu versions may not have the most recent version of mlpack -available---for instance, at the time of this writing, Ubuntu 16.04 only has -mlpack 2.0.1 available. Options include upgrading Ubuntu to a newer release, -finding a PPA or other non-official sources, or installing with a manual build -(below). - -If mlpack is not available in your system's package manager, then you can follow -this document for how to compile and install mlpack from source. - -mlpack uses CMake as a build system and allows several flexible build -configuration options. One can consult any of numerous CMake tutorials for -further documentation, but this tutorial should be enough to get mlpack built -and installed on most Linux and UNIX-like systems (including OS X). If you want -to build mlpack on Windows, see \ref build_windows (alternatively, you can read -Keon's excellent tutorial which -is based on older versions). - -You can download the latest mlpack release from here: -mlpack-3.4.2 - -@section build_simple Simple Linux build instructions - -Assuming all dependencies are installed in the system, you can run the commands -below directly to build and install mlpack. - -@code -$ wget https://www.mlpack.org/files/mlpack-3.4.2.tar.gz -$ tar -xvzpf mlpack-3.4.2.tar.gz -$ mkdir mlpack-3.4.2/build && cd mlpack-3.4.2/build -$ cmake ../ -$ make -j4 # The -j is the number of cores you want to use for a build. -$ sudo make install -@endcode - -If the \c cmake \c .. command fails, you are probably missing a dependency, so -check the output and install any necessary libraries. (See \ref build_dep.) - -@note If you are using RHEL7/CentOS 7, the default version of gcc is too old. -One solution is to use \c devtoolset-8; more information is available at -https://www.softwarecollections.org/en/scls/rhscl/devtoolset-8/ . - -On many Linux systems, mlpack will install by default to @c /usr/local/lib and -you may need to set the @c LD_LIBRARY_PATH environment variable: - -@code -export LD_LIBRARY_PATH=/usr/local/lib -@endcode - -The instructions above are the simplest way to get, build, and install mlpack. -The sections below discuss each of those steps in further detail and show how to -configure mlpack. - -@section build_builddir Creating Build Directory - -First we should unpack the mlpack source and create a build directory. - -@code -$ tar -xvzpf mlpack-3.4.2.tar.gz -$ cd mlpack-3.4.2 -$ mkdir build -@endcode - -The directory can have any name, not just 'build', but 'build' is sufficient. - -@section build_dep Dependencies of mlpack - -mlpack depends on the following libraries, which need to be installed on the -system and have headers present: - - - Armadillo >= 9.800 (with LAPACK support) - - cereal >= 1.1.2 - - ensmallen >= 2.10.0 (will be downloaded if not found) - -In addition, mlpack has the following optional dependencies: - - - STB: this will allow loading of images; the library is downloaded if not - found and the CMake variable DOWNLOAD_STB_IMAGE is set to ON (the default) - -For Python bindings, the following packages are required: - - - setuptools - - cython >= 0.24 - - numpy - - pandas >= 0.15.0 - - pytest-runner - -In Ubuntu (>= 18.04) and Debian (>= 10) all of these dependencies can be -installed through apt: - -@code -# apt-get install libcereal-dev libarmadillo-dev binutils-dev python3-pandas - python3-numpy cython3 python3-setuptools -@endcode - -If you are using Ubuntu 19.10 or newer, you can also install @c libensmallen-dev -and @c libstb-dev, so that CMake does not need to automatically download those -packages: - -@code -# apt-get install libensmallen-dev libstb-dev -@endcode - -@note For older versions of Ubuntu and Debian, Armadillo needs to be built from -source as apt installs an older version. So you need to omit -\c libarmadillo-dev from the code snippet above and instead use -this link - to download the required file. Extract this file and follow the README in the - uncompressed folder to build and install Armadillo. - -On Fedora, Red Hat, or CentOS, these same dependencies can be obtained via dnf: - -@code -# dnf install armadillo-devel binutils-devel python3-Cython python3-setuptools - python3-numpy python3-pandas ensmallen-devel stbi-devel cereal-devel -@endcode - -(It's also possible to use python3 packages from the package manager---mlpack -will work with either. Also, the ensmallen-devel package is only available in -Fedora 29 or RHEL7 or newer.) - -@section build_config Configuring CMake - -Running CMake is the equivalent to running `./configure` with autotools. If you -run CMake with no options, it will configure the project to build without -debugging or profiling information (for speed). - -@code -$ cd build -$ cmake ../ -@endcode - -You can manually specify options to compile with debugging information and -profiling information (useful if you are developing mlpack): - -@code -$ cd build -$ cmake -D DEBUG=ON -D PROFILE=ON ../ -@endcode - -The full list of options mlpack allows: - - - DEBUG=(ON/OFF): compile with debugging symbols (default OFF) - - PROFILE=(ON/OFF): compile with profiling symbols (default OFF) - - ARMA_EXTRA_DEBUG=(ON/OFF): compile with extra Armadillo debugging symbols - (default OFF) - - BUILD_TESTS=(ON/OFF): compile the \c mlpack_test program when `make` is run - (default ON) - - BUILD_CLI_EXECUTABLES=(ON/OFF): compile the mlpack command-line executables - (i.e. \c mlpack_knn, \c mlpack_kfn, \c mlpack_logistic_regression, etc.) - (default ON) - - BUILD_PYTHON_BINDINGS=(ON/OFF): compile the bindings for Python, if the - necessary Python libraries are available (default OFF) - - BUILD_R_BINDINGS=(ON/OFF): compile the bindings for R, if R is found - (default OFF) - - BUILD_GO_BINDINGS=(ON/OFF): compile Go bindings, if Go and the necessary Go - and Gonum exist. (default OFF) - - BUILD_JULIA_BINDINGS=(ON/OFF): compile Julia bindings, if Julia is found - (default OFF) - - BUILD_SHARED_LIBS=(ON/OFF): compile shared libraries and executables as opposed to - static libraries (default ON) - - TEST_VERBOSE=(ON/OFF): run test cases in \c mlpack_test with verbose output - (default OFF) - - DISABLE_DOWNLOADS=(ON/OFF): Disable downloads of dependencies during build - (default OFF) - - PYTHON_EXECUTABLE=(/path/to/python_version): Path to specific Python executable - - PYTHON_INSTALL_PREFIX=(/path/to/python/): Path to root of Python installation - - JULIA_EXECUTABLE=(/path/to/julia): Path to specific Julia executable - - BUILD_MARKDOWN_BINDINGS=(ON/OFF): Build Markdown bindings for website - documentation (default OFF) - - BUILD_DOCS=(ON/OFF): build Doxygen documentation, if Doxygen is available - (default ON) - - MATHJAX=(ON/OFF): use MathJax for generated Doxygen documentation (default - OFF) - - USE_OPENMP=(ON/OFF): if ON, then use OpenMP if the compiler supports it; if - OFF, OpenMP support is manually disabled (default ON) - -Each option can be specified to CMake with the '-D' flag. Other tools can also -be used to configure CMake, but those are not documented here. - -For example, if you would like to build mlpack and its CLI bindings statically, then -you need to execute the following commands: - -@code -$ cd build -$ cmake -D BUILD_SHARED_LIBS=OFF ../ -@endcode - -In addition, the following directories may be specified, to find include files -and libraries. These also use the '-D' flag. - - - ARMADILLO_INCLUDE_DIR=(/path/to/armadillo/include/): path to Armadillo headers - - ARMADILLO_LIBRARY=(/path/to/armadillo/libarmadillo.so): location of Armadillo - library - - CEREAL_INCLUDE_DIR=(/path/to/cereal/include): path to include directory for - cereal - - ENSMALLEN_INCLUDE_DIR=(/path/to/ensmallen/include): path to include directory - for ensmallen - - STB_IMAGE_INCLUDE_DIR=(/path/to/stb/include): path to include directory for - STB image library - - MATHJAX_ROOT=(/path/to/mathjax): path to root of MathJax installation - -@section build_build Building mlpack - -Once CMake is configured, building the library is as simple as typing 'make'. -This will build all library components. - -@code -$ make -@endcode - -It's often useful to specify \c -jN to the \c make command, which will build on -\c N processor cores. That can accelerate the build significantly. Sometimes -using many cores may exhaust the memory so choose accordingly. - -You can specify individual components which you want to build, if you do not -want to build everything in the library: - -@code -$ make mlpack_pca mlpack_knn mlpack_kfn -@endcode - -One particular component of interest is mlpack_test, which runs the mlpack test -suite. This is not built when @c make is run. You can build this component -with - -@code -$ make mlpack_test -@endcode - -We use Catch2 to write our tests. -To run all tests, you can simply use CTest: - -@code -$ ctest . -@endcode - -Or, you can run the test suite manually: - -@code -$ bin/mlpack_test -@endcode - -To run all tests in a particular file you can run: - -@code -$ ./bin/mlpack_test "[testname]" -@endcode - -where testname is the name of the test suite. -For example to run all collaborative filtering tests implemented in cf_test.cpp you can run: - -@code -./bin/mlpack_test "[CFTest]" -@endcode - -Now similarly you can run all the binding related tests using: - -@code -./bin/mlpack_test "[BindingTests]" -@endcode - -To run a single test, you can explicitly provide the name of the test; for example, -to run BinaryClassificationMetricsTest implemented in cv_test.cpp you can run the following: - -@code -./bin/mlpack_test BinaryClassificationMetricsTest -@endcode - -If the build fails and you cannot figure out why, register an account on Github -and submit an issue and the mlpack developers will quickly help you figure it -out: - -https://mlpack.org/ - -https://github.com/mlpack/mlpack - -Alternately, mlpack help can be found in IRC at \#mlpack on chat.freenode.net. - -@section install Installing mlpack - -If you wish to install mlpack to the system, make sure you have root privileges -(or write permissions to those two directories), and simply type - -@code -# make install -@endcode - -You can now run the executables by name; you can link against mlpack with -\c -lmlpack, and the mlpack headers are found in \c /usr/include or -\c /usr/local/include (depending on the system and CMake configuration). If -Python bindings were installed, they should be available when you start Python. - -@section build_run Using mlpack without installing - -If you would prefer to use mlpack after building but without installing it to -the system, this is possible. All of the command-line programs in the -@c build/bin/ directory will run directly with no modification. - -For running the Python bindings from the build directory, the situation is a -little bit different. You will need to set the following environment variables: - -@code -export LD_LIBRARY_PATH=/path/to/mlpack/build/lib/:${LD_LIBRARY_PATH} -export PYTHONPATH=/path/to/mlpack/build/src/mlpack/bindings/python/:${PYTHONPATH} -@endcode - -(Be sure to substitute the correct path to your build directory for -`/path/to/mlpack/build/`.) - -Once those environment variables are set, you should be able to start a Python -interpreter and `import mlpack`, then use the Python bindings. - -*/ diff --git a/doc/guide/build_windows.hpp b/doc/guide/build_windows.hpp deleted file mode 100644 index c85379492e..0000000000 --- a/doc/guide/build_windows.hpp +++ /dev/null @@ -1,244 +0,0 @@ -/** - * @file build_windows.hpp - * @author German Lancioni - * @author Miguel Canteras - * @author Shikhar Jaiswal - * @author Ziyang Jiang - -@page build_windows Building mlpack From Source on Windows - -@section build_windows_intro Introduction - -This tutorial will show you how to build mlpack for Windows from source, so -you can later create your own C++ applications, using two different ways: - - - Using CMake to generate an intermeditate Visual Studio solution (`.sln`). - - @ref build_visual_studio_cmake_integration "Use Visual Studio's CMake integration to directly build from the `CMakeLists`." - -Before you try building mlpack, you may -want to install mlpack using vcpkg for Windows. If you don't want to install -using vcpkg, skip this section and continue with the build tutorial. - -- Install Git (https://git-scm.com/downloads and execute setup) - -- Install CMake (https://cmake.org/ and execute setup) - -- Install vcpkg (https://github.com/Microsoft/vcpkg and execute setup) - -- To install the mlpack library only: - -@code -PS> .\vcpkg install mlpack:x64-windows -@endcode - -- To install mlpack and its console programs: -@code -PS> .\vcpkg install mlpack[tools]:x64-windows -@endcode - -After installing, in Visual Studio, you can create a new project (or open -an existing one). The library is immediately ready to be included -(via preprocessor directives) and used in your project without additional -configuration. - -@section build_windows_env Build Environment - -This tutorial has been designed and tested using: -- Windows 10 -- Visual Studio 2019 (toolset v142) -- mlpack -- OpenBLAS.0.2.14.1 -- armadillo (newest version) -- and x64 configuration - -The directories and paths used in this tutorial are just for reference purposes. - -@section build_windows_prereqs Pre-requisites - -- Install CMake for Windows (win64-x64 version from https://cmake.org/download/) -and make sure you can use it from the Command Prompt (may need to add the PATH to -system environment variables or manually set the PATH before running CMake) - -- Download the latest mlpack release from here: -mlpack website - -@section build_windows_instructions Windows build instructions - -- Unzip mlpack to "C:\mlpack\mlpack" -- Open Visual Studio and select: File > New > Project from Existing Code - - Type of project: Visual C++ - - Project location: "C:\mlpack\mlpack" - - Project name: mlpack - - Finish -- Make sure the solution configuration is "Debug" and the solution platform is "x64" for this Visual Studio project -- We will use this Visual Studio project to get the OpenBLAS dependency in the next section - -@section build_windows_dependencies Dependencies - - OpenBLAS Dependency - -- Open the NuGet packages manager (Tools > NuGet Package Manager > Manage NuGet Packages for Solution...) -- Click on the “Browse” tab and search for “openblas” -- Click on OpenBlas and check the mlpack project, then click Install -- Once it has finished installing, close Visual Studio - - Building OpenBLAS from Source - -Unfortunately, the support for building `LAPACK` and `BLAS` on Windows is quite poor, due to the need for Fortran -compiler and libraries. The easiest method to get the necessary `BLAS/LAPACK` libraries built on Windows is to -compile OpenBLAS with LLVM's `clang-cl` and `flang` to produce the required static library (`.lib`) files -compatible with the MSVC compiler. A comprehensive guide on the -compilation -of OpenBLAS for Windows can be found here. - -One could always download prebuilt `LAPACK` and `BLAS` libraries for Windows. However, there are few official -sources, and some of those libraries may require further `dll`s at runtime which may not be available in your -system. - -It you choose to build `OpenBLAS` from source, make sure that `LAPACK` functions are also built. Finally, make -sure that the `openblas.lib` library is linked in your `Armadillo` build (see below), as well as the library -path used for the CMake options `BLAS_LIBRARIES` and `LAPACK_LIBRARIES` in the mlpack CMake project. - - Armadillo Dependency - -- Download the newest version of Armadillo from Sourceforge -- Unzip to "C:\mlpack\armadillo" -- Create a "build" directory into "C:\mlpack\armadillo\" -- Open the Command Prompt and navigate to "C:\mlpack\armadillo\build" -- Run cmake: - -@code -cmake -G "Visual Studio 16 2019" -A x64 -DBLAS_LIBRARY:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARY:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" .. -@endcode - -@note If you are using different directory paths, a different configuration (e.g. Release) -or a different VS version, update the cmake command accordingly. If CMake cannot identify the -compiler version, check if the Visual Studio compiler and Windows SDK are installed correctly. - -- Once it has successfully finished, open "C:\mlpack\armadillo\build\armadillo.sln" -- Build > Build Solution -- Once it has successfully finished, close Visual Studio - -@section build_windows_mlpack Building mlpack with CMake-Generated Solution - -- Create a "build" directory into "C:\mlpack\mlpack\" -- You can generate the project using either cmake via command line or GUI. If you prefer to use GUI, refer to the \ref build_windows_appendix "appendix" -- To use the CMake command line prompt, open the Command Prompt and navigate to "C:\mlpack\mlpack\build" -- Run cmake: - -@code -cmake -G "Visual Studio 16 2019" -A x64 -DBLAS_LIBRARIES:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARIES:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DARMADILLO_INCLUDE_DIR="C:/mlpack/armadillo/include" -DARMADILLO_LIBRARY:FILEPATH="C:/mlpack/armadillo/build/Debug/armadillo.lib" -DDEBUG=OFF -DPROFILE=OFF .. -@endcode - -@note cmake will attempt to automatically download the ensmallen dependency. If for some reason cmake can't download the dependency, you will need to manually download ensmallen from http://ensmallen.org/ and extract it to "C:\mlpack\mlpack\deps\". Then, specify the path to ensmallen using the flag: -DENSMALLEN_INCLUDE_DIR=C:/mlpack/mlpack/deps/ensmallen/include - -- Once CMake configuration has successfully finished, open "C:\mlpack\mlpack\build\mlpack.sln" -- Build > Build Solution (this may be by default in Debug mode) -- Once it has sucessfully finished, you will find the library files you need in: "C:\mlpack\mlpack\build\Debug" (or "C:\mlpack\mlpack\build\Release" if you changed to Release mode) - -You are ready to create your first application, take a look at the @ref sample_ml_app "Sample C++ ML App" - -@section build_visual_studio_cmake_integration Building mlpack with Visual Studio's CMake Integration - -This project can be directly built from the `CMakeLists.txt` with the latest version of MS Visual Studio, -given you have CMake integration via the -C++ -CMake tools for Windows. To open the CMake project with Visual Studio, select File->Open->CMake -in the top menu, followed by selecting the root `CMakeLists.txt` located in mlpack's root directory. - -In order to allow Visual Studio to configure the CMake project, the CMake configuration json will have -to be edited to provide the relevant options -shown in the `README` needed to find all the dependencies. The options that you -must provide to Visual Studio's CMake are: - - - `ARMADILLO_INCLUDE_DIR` - - `ARMADILLO_LIBRARY` - - `CEREAL_INCLUDE_DIR` - - `BLAS_LIBRARIES` - - `LAPACK_LIBRARIES` - -The CMake configuration json can be editted in Visual Studio by right clicking the root `CMakeLists.txt` -in the project view, selecting CMake settings for mlpack and finally clicking on edit JSON. -Adding a new CMake option can be done by adding object fields with the following format to the variables -array in the `CMakeSettings.json`: - -@code -{ - "name": "options_name_string", - "value": "options_value_string", - "type" : "{BOOL|FILEPATH|PATH|STRING}" -} -@endcode - -Here is a full example of the `CMakeSettings.json`file: - -@code -{ - "configurations": [ - { - "name": "x64-Debug (default)", - "generator": "Ninja", - "configurationType": "Debug", - "inheritEnvironments": [ "msvc_x64_x64" ], - "buildRoot": "${projectDir}\\out\\build\\${name}", - "installRoot": "${projectDir}\\out\\install\\${name}", - "cmakeCommandArgs": "", - "buildCommandArgs": "", - "ctestCommandArgs": "", - "variables": [ - { - "name": "ARMADILLO_INCLUDE_DIR", - "value": "PATH/TO/CPP/DEPENDENCY/armadillo-10.1.2/include", - "type": "PATH" - }, - { - "name": "ARMADILLO_LIBBRARY", - "value": "PATH/TO/CPP/DEPENDENCY/armadillo-10.1.2/lib/armadillo.lib", - "type": "PATH" - }, - { - "name": "CEREAL_INCLUDE_DIR", - "value": "PATH/TO/CPP/DEPENDENCY/cereal-1.3.0/include", - "type": "PATH" - }, - { - "name": "BLAS_LIBRARIES", - "value": "PATH/TO/CPP/DEPENDENCY/OpenBLAS/lib/openblas.lib", - "type": "PATH" - }, - { - "name": "LAPACK_LIBRARIES", - "value": "PATH/TO/CPP/DEPENDENCY/OpenBLAS/lib/openblas.lib", - "type": "PATH" - } - ] - } - ] -} -@endcode - -@section build_windows_appendix Appendix - -If you prefer to use cmake GUI, follow these instructions: - - - To use the CMake GUI, open "CMake". - - For "Where is the source code:" set `C:\mlpack\mlpack\` - - For "Where to build the binaries:" set `C:\mlpack\mlpack\build` - - Click `Configure` - - If there is an error and Armadillo is not found, try "Add Entry" with the - following variables and reconfigure: - - Name: `ARMADILLO_INCLUDE_DIR`; type `PATH`; value `C:/mlpack/armadillo/include/` - - Name: `ARMADILLO_LIBRARY`; type `FILEPATH`; value `C:/mlpack/armadillo/build/Debug/armadillo.lib` - - Name: `BLAS_LIBRARY`; type `FILEPATH`; value `C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a` - - Name: `LAPACK_LIBRARY`; type `FILEPATH`; value `C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a` - - Once CMake has configured successfully, hit "Generate" to create the `.sln` file. - -@section build_windows_additional_information Additional Information - -If you are facing issues during the build process of mlpack, you may take a look at other third-party tutorials for Windows, but they may be out of date: - - * Github wiki Windows Build page
- * Keon's tutorial for mlpack 2.0.3
- * Kirizaki's tutorial for mlpack 2
- -*/ diff --git a/doc/guide/cv.hpp b/doc/guide/cv.hpp deleted file mode 100644 index 4c6f8f9478..0000000000 --- a/doc/guide/cv.hpp +++ /dev/null @@ -1,372 +0,0 @@ -namespace mlpack { -namespace cv { - -/*! @page cv Cross-Validation - -@section cvintro Introduction - -@b mlpack implements cross-validation support for its learning algorithms, for a -variety of performance measures. Cross-validation is useful for determining an -estimate of how well the learner will generalize to un-seen test data. It is a -commonly used part of the data science pipeline. - -In short, given some learner and some performance measure, we wish to get an -average of the performance measure given different splits of the dataset into -training data and validation data. The learner is trained on the training data, -and the performance measure is evaluated on the validation data. - -mlpack currently implements two easy-to-use forms of cross-validation: - - - @b simple @b cross-validation, where we simply desire the performance measure - on a single split of the data into a training set and validation set - - - @b k-fold @b cross-validation, where we split the data k ways and desire the - average performance measure on each of the k splits of the data - -In this tutorial we will see the usage examples and details of the -cross-validation module. Because the cross-validation code is generic and can -be used with any learner and performance measure, any use of the -cross-validation code in mlpack has to be in C++. - -This tutorial is split into the following sections: - - - @ref cvbasic Simple cross-validation examples - - @ref cvbasic_ex_1 10-fold cross-validation on softmax regression - - @ref cvbasic_ex_2 10-fold cross-validation on weighted decision trees - - @ref cvbasic_ex_3 10-fold cross-validation with categorical decision trees - - @ref cvbasic_ex_4 Simple cross-validation for linear regression - - @ref cvbasic_metrics Performance measures - - @ref cvbasic_api The \c KFoldCV and \c SimpleCV classes - - @ref cvbasic_further Further reference - -@section cvbasic Simple cross-validation examples - -@subsection cvbasic_ex_1 10-fold cross-validation on softmax regression - -Suppose we have some data to train and validate on, as defined below: - -@code - // 100-point 6-dimensional random dataset. - arma::mat data = arma::randu(6, 100); - // Random labels in the [0, 4] interval. - arma::Row labels = - arma::randi>(100, arma::distr_param(0, 4)); - size_t numClasses = 5; -@endcode - -The code above generates an 100-point random 6-dimensional dataset with 5 -classes. - -To run 10-fold cross-validation for softmax regression with accuracy as a -performance measure, we can write the following piece of code. - -@code - KFoldCV cv(10, data, labels, numClasses); - double lambda = 0.1; - double softmaxAccuracy = cv.Evaluate(lambda); -@endcode - -Note that the \c Evaluate method of \c KFoldCV takes any hyperparameters of an -algorithm---that is, anything that is not \c data, \c labels, \c numClasses, -\c datasetInfo, or \c weights (those last three may not be present for every -algorithm type). To be more specific, in this example the \c Evaluate method -relies on the following \ref regression::SoftmaxRegression "SoftmaxRegression" -constructor: - -@code - template - SoftmaxRegression(const arma::mat& data, - const arma::Row& labels, - const size_t numClasses, - const double lambda = 0.0001, - const bool fitIntercept = false, - OptimizerType optimizer = OptimizerType()); -@endcode - -which has the parameter \c lambda after three conventional arguments (\c data, -\c labels and \c numClasses). We can skip passing \c fitIntercept and \c -optimizer since there are the default values. (Technically, we don't even need -to pass \c lambda since there is a default value.) - -In general to cross-validate you need to specify what machine learning algorithm -and metric you are going to use, and then to pass some conventional data-related -parameters into one of the cross-validation constructors and all other -parameters (which are generally hyperparameters) into the \c Evaluate method. - -@subsection cvbasic_ex_2 10-fold cross-validation on weighted decision trees - -In the following example we will cross-validate -\ref tree::DecisionTree "DecisionTree" with weights. This is very similar to -the previous example, except that we also have instance weights for each point -in the dataset. We can generate weights for the dataset from the previous -example with the code below: - -@code - // Random weights for every point from the code snippet above. - arma::rowvec weights = arma::randu(1, 100); -@endcode - -Given those weights for each point, we can now perform cross-validation by also -passing the weights to the constructor of \c KFoldCV: - -@code - KFoldCV, Accuracy> cv2(10, data, labels, numClasses, weights); - size_t minimumLeafSize = 8; - double weightedDecisionTreeAccuracy = cv2.Evaluate(minimumLeafSize); -@endcode - -As with the previous example, internally this call to \c cv2.Evaluate() relies -on the following \ref tree::DecisionTree "DecisionTree" constructor: - -@code - template - DecisionTree(MatType&& data, - LabelsType&& labels, - const size_t numClasses, - WeightsType&& weights, - const size_t minimumLeafSize = 10, - const std::enable_if_t::type>::value>* - = 0); -@endcode - -@subsection cvbasic_ex_3 10-fold cross-validation with categorical decision trees - -\ref tree::DecisionTree "DecisionTree" models can be constructed in multiple -other ways. For example, if we have a dataset with both categorical and -numerical features, we can also perform cross-validation by using the associated -\c data::DatasetInfo object. Thus, given some \c data::DatasetInfo object -called \c datasetInfo (that perhaps was produced by a call to \c data::Load() ), -we can perform k-fold cross-validation in a similar manner to the other -examples: - -@code - KFoldCV, Accuracy> cv3(10, data, datasetInfo, labels, - numClasses); - double decisionTreeWithDIAccuracy = cv3.Evaluate(minimumLeafSize); -@endcode - -This particular call to \c cv3.Evaluate() relies on the following -\ref tree::DecisionTree "DecisionTree" constructor: - -@code - template - DecisionTree(MatType&& data, - const data::DatasetInfo& datasetInfo, - LabelsType&& labels, - const size_t numClasses, - const size_t minimumLeafSize = 10); -@endcode - -@subsection cvbasic_ex_4 Simple cross-validation for linear regression - -\c SimpleCV has the same interface as \c KFoldCV, except it takes as one of its -arguments a proportion (from 0 to 1) of data used as a validation set. For -example, to validate \ref regression::LinearRegression "LinearRegression" with -20\% of the data used in the validation set we can write the following code. - -@code - // Random responses for every point from the code snippet in the beginning of - // the tutorial. - arma::rowvec responses = arma::randu(100); - - SimpleCV cv4(0.2, data, responses); - double lrLambda = 0.05; - double lrMSE = cv4.Evaluate(lrLambda); -@endcode - -@section cvbasic_metrics Performance measures - -The cross-validation classes require a performance measure to be specified. -\b mlpack has a number of performance measures implemented; below is a list: - - - mlpack::cv::Accuracy: a simple measure of accuracy - - mlpack::cv::F1: the F1 score; depends on an averaging strategy - - mlpack::cv::MSE: minimum squared error (for regression problems) - - mlpack::cv::Precision: the precision, for classification problems - - mlpack::cv::Recall: the recall, for classification problems - -In addition, it is not difficult to implement a custom performance measure. A -class following the structure below can be used: - -@code -class CustomMeasure -{ - // - // This evaluates the metric given a trained model and a set of data (with - // labels or responses) to evaluate on. The data parameter will be a type of - // Armadillo matrix, and the labels will be the labels that go with the model. - // - // If you know that your model is a classification model (and thus that - // ResponsesType will be arma::Row), it is ok to replace the - // ResponsesType template parameter with arma::Row. - // - template - static double Evaluate(MLAlgorithm& model, - const DataType& data, - const ResponsesType& labels) - { - // Inside the method you should call model.Predict() and compare the - // values with the labels, in order to get the desired performance measure - // and return it. - } -}; -@endcode - -Once this is implemented, then \c CustomMeasure (or whatever the class is -called) is easy to use as a custom performance measure with \c KFoldCV or -\c SimpleCV. - -@section cvbasic_api The KFoldCV and SimpleCV classes - -This section provides details about the \c KFoldCV and \c SimpleCV classes. -The cross-validation infrastructure is based on heavy amounts of template -metaprogramming, so that any \b mlpack learner and any performance measure can -be used. Both classes have two required template parameters and one optional -parameter: - - - \c MLAlgorithm: the type of learner to be used - - \c Metric: the performance measure to be evaluated - - \c MatType: the type of matrix used to store the data - -In addition, there are two more template parameters, but these are automatically -extracted from the given \c MLAlgorithm class, and users should not need to -specify these parameters except when using an unconventional type like -\c arma::fmat for data points. - -The general structure of the \c KFoldCV and \c SimpleCV classes is split into -two parts: - - - The constructor: create the object, and store the data for the \c MLAlgorithm - training. - - The \c Evaluate() method: take any non-data parameters for the - \c MLAlgorithm and calculate the desired performance measure. - -This split is important because it defines the API: all data-related parameters -are passed to the constructor, whereas algorithm hyperparameters are passed to -the \c Evaluate() method. - -@subsection cvbasic_api_constructor The KFoldCV and SimpleCV constructors - -There are six constructors available for \c KFoldCV and \c SimpleCV, each -tailored for a different learning situation. Each is given below for the -\c KFoldCV class, but the same constructors are also available for the -\c SimpleCV class, with the exception that instead of specifying \c k, the -number of folds, the \c SimpleCV class takes a parameter between 0 and 1 -specifying the percentage of the dataset to use as a validation set. - - - `KFoldCV(k, xs, ys)`: this is for unweighted regression applications and - two-class classification applications; \c xs is the dataset and \c ys - are the responses or labels for each point in the dataset. - - - `KFoldCV(k, xs, ys, numClasses)`: this is for unweighted classification - applications; \c xs is the dataset, \c ys are the class labels for each - data point, and \c numClasses is the number of classes in the dataset. - - - `KFoldCV(k, xs, datasetInfo, ys, numClasses)`: this is for unweighted - categorical/numeric classification applications; \c xs is the dataset, - \c datasetInfo is a data::DatasetInfo object that holds the types of - each dimension in the dataset, \c ys are the class labels for each data - point, and \c numClasses is the number of classes in the dataset. - - - `KFoldCV(k, xs, ys, weights)`: this is for weighted regression or - two-class classification applications; \c xs is the dataset, \c ys are - the responses or labels for each point in the dataset, and \c weights - are the weights for each point in the dataset. - - - `KFoldCV(k, xs, ys, numClasses, weights)`: this is for weighted - classification applications; \c xs is the dataset, \c ys are the class - labels for each point in the dataset; \c numClasses is the number of - classes in the dataset, and \c weights holds the weights for each point - in the dataset. - - - `KFoldCV(k, xs, datasetInfo, ys, numClasses, weights)`: this is for - weighted cateogrical/numeric classification applications; \c xs is the - dataset, \c datasetInfo is a data::DatasetInfo object that holds the - types of each dimension in the dataset, \c ys are the class labels for - each data point, \c numClasses is the number of classes in each dataset, - and \c weights holds the weights for each point in the dataset. - -Note that the constructor you should use is the constructor that most closely -matches the constructor of the machine learning algorithm you would like -performance measures of. So, for instance, if you are doing multi-class softmax -regression, you could call the constructor -\c "SoftmaxRegression(xs, ys, numClasses)". Therefore, for \c KFoldCV you would -call the constructor \c "KFoldCV(k, xs, ys, numClasses)" and for \c SimpleCV you -would call the constructor \c "SimpleCV(pct, xs, ys, numClasses)". - -@subsection cvbasic_api_evaluate The Evaluate() method - -The other method that \c KFoldCV and \c SimpleCV have is the method to -actually calculate the performance measure: \c Evaluate(). The \c Evaluate() -method takes any hyperparameters that would follow the data arguments to the -constructor or \c Train() method of the given \c MLAlgorithm. The -\c Evaluate() method takes no more arguments than that, and returns the -desired performance measure on the dataset. - -Therefore, let us suppose that we are interested in cross-validating the -performance of a softmax regression model, and that we have constructed -the appropriate \c KFoldCV object using the code below: - -@code -KFoldCV cv(k, data, labels, numClasses); -@endcode - -The \ref regression::SoftmaxRegression "SoftmaxRegression" class has the -constructor - -@code - template - SoftmaxRegression(const arma::mat& data, - const arma::Row& labels, - const size_t numClasses, - const double lambda = 0.0001, - const bool fitIntercept = false, - OptimizerType optimizer = OptimizerType()); -@endcode - -Note that all parameters after are \c numClasses are optional. This means that -we can specify none or any of them in our call to \c Evaluate(). Below is some -example code showing three different ways we can call \c Evaluate() with the -\c cv object from the code snippet above. - -@code -// First, call with all defaults. -double result1 = cv.Evaluate(); - -// Next, call with lambda set to 0.1 and fitIntercept set to true. -double result2 = cv.Evaluate(0.1, true); - -// Lastly, create a custom optimizer to use for optimization, and use a lambda -// value of 0.5 and fit no intercept. -optimization::SGD<> sgd(0.05, 50000); // Step size of 0.05, 50k max iterations. -double result3 = cv.Evaluate(0.5, false, sgd); -@endcode - -The same general idea applies to any \c MLAlgorithm: all hyperparameters must be -passed to the \c Evaluate() method of \c KFoldCV or \c SimpleCV. - -@section cvbasic_further Further references - -For further documentation, please see the associated Doxygen documentation for -each of the relevant classes: - - - mlpack::cv::SimpleCV - - mlpack::cv::KFoldCV - - mlpack::cv::Accuracy - - mlpack::cv::F1 - - mlpack::cv::MSE - - mlpack::cv::Precision - - mlpack::cv::Recall - -If you are interested in implementing a different cross-validation strategy than -k-fold cross-validation or simple cross-validation, take a look at the -implementations of each of those classes to guide your implementation. - -In addition, the @ref hpt "hyperparameter tuner" documentation may also be -relevant. - -*/ - -} // namespace cv -} // namespace mlpack diff --git a/doc/guide/hpt.hpp b/doc/guide/hpt.hpp deleted file mode 100644 index 3d87d36d6b..0000000000 --- a/doc/guide/hpt.hpp +++ /dev/null @@ -1,238 +0,0 @@ -namespace mlpack { -namespace hpt { - -/*! @page hpt_guide Hyper-Parameter Tuning - -@section hptintro Introduction - -\b mlpack implements a generic hyperparameter tuner that is able to tune both -continuous and discrete parameters of various different algorithms. This is an -important task---the performance of many machine learning algorithms can be -highly dependent on the hyperparameters that are chosen for that algorithm. -(One example: the choice of \f$k\f$ for a \f$k\f$-nearest-neighbors classifier.) - -This hyper-parameter tuner is built on the same general concept as the -cross-validation classes (see the @ref cv "cross-validation tutorial"): given -some machine learning algorithm, some data, some performance measure, and a set -of hyperparameters, attempt to find the hyperparameter set that best optimizes -the performance measure on the given data with the given algorithm. - -\b mlpack's implementation of hyperparameter tuning is flexible, and is built in -a way that supports many algorithms and many optimizers. At the time of this -writing, complex hyperparameter optimization techniques are not available, but -the hyperparameter tuner does support these, should they be implemented in the -future. - -In this tutorial we will see the usage examples of the hyper-parameter tuning -module, and also more details about the \c HyperParameterTuner class. - -@section hptbasic Basic Usage - -The interface of the hyper-parameter tuning module is quite similar to the -interface of the @ref cv "cross-validation module". To construct a \c -HyperParameterTuner object you need to specify as template parameters what -machine learning algorithm, cross-validation strategy, performance measure, and -optimization strategy (\c ens::GridSearch will be used by -default) you are going to use. Then, you must pass the same arguments as for -the cross-validation classes: the data and labels (or responses) to use are -given to the constructor, and the possible hyperparameter values are given to -the \c HyperParameterTuner::Optimize() method, which returns the best -algorithm configuration as a \c std::tuple<>. - -Let's see some examples. - -Suppose we have the following data to train and validate on. -@code - // 100-point 5-dimensional random dataset. - arma::mat data = arma::randu(5, 100); - // Noisy responses retrieved by a random linear transformation of data. - arma::rowvec responses = arma::randu(5) * data + - 0.1 * arma::randn(100); -@endcode - -Given the dataset above, we can use the following code to try to find a good \c -lambda value for \ref regression::LinearRegression "LinearRegression". Here we -use \ref cv::SimpleCV "SimpleCV" instead of k-fold cross-validation to save -computation time. - -@code - // Using 80% of data for training and remaining 20% for assessing MSE. - double validationSize = 0.2; - HyperParameterTuner hpt(validationSize, - data, responses); - - // Finding a good value for lambda from the discrete set of values 0.0, 0.001, - // 0.01, 0.1, and 1.0. - arma::vec lambdas{0.0, 0.001, 0.01, 0.1, 1.0}; - double bestLambda; - std::tie(bestLambda) = hpt.Optimize(lambdas); -@endcode - -In this example we have used \c ens::GridSearch (the -default optimizer) to find a good value for the \c lambda hyper-parameter. For -that we have specified what values should be tried. - -@section hptfixed Fixed Arguments - -When some hyper-parameters should not be optimized, you can specify values -for them with the \c Fixed() method as in the following example of trying to -find good \c lambda1 and \c lambda2 values for \ref regression::LARS "LARS" -(least-angle regression). - -@code - HyperParameterTuner hpt2(validationSize, data, - responses); - - // The hyper-parameter tuner should not try to change the transposeData or - // useCholesky parameters. - bool transposeData = true; - bool useCholesky = false; - - // We wish only to search for the best lambda1 and lambda2 values. - arma::vec lambda1Set{0.0, 0.001, 0.01, 0.1, 1.0}; - arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0}; - - double bestLambda1, bestLambda2; - std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Fixed(transposeData), - Fixed(useCholesky), lambda1Set, lambda2Set); -@endcode - -Note that for the call to \c hpt2.Optimize(), we have used the same order of -arguments as they appear in the corresponding \ref regression::LARS "LARS" -constructor: - -@code - LARS(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData = true, - const bool useCholesky = false, - const double lambda1 = 0.0, - const double lambda2 = 0.0, - const double tolerance = 1e-16); -@endcode - -@section hptgradient Gradient-Based Optimization - -In some cases we may wish to optimize a hyperparameter over the space of all -possible real values, instead of providing a grid in which to search. -Alternately, we may know approximately optimal values from a grid search for -real-valued hyperparameters, but wish to further tune those values. - -In this case, we can use a gradient-based optimizer for hyperparameter search. -In the following example, we try to optimize the \c lambda1 and \c lambda2 -hyper-parameters for \ref regression::LARS "LARS" with the -\c ens::GradientDescent optimizer. - -@code - HyperParameterTuner hpt3(validationSize, - data, responses); - - // GradientDescent can be adjusted in the following way. - hpt3.Optimizer().StepSize() = 0.1; - hpt3.Optimizer().Tolerance() = 1e-15; - - // We can set up values used for calculating gradients. - hpt3.RelativeDelta() = 0.01; - hpt3.MinDelta() = 1e-10; - - double initialLambda1 = 0.001; - double initialLambda2 = 0.002; - - double bestGDLambda1, bestGDLambda2; - std::tie(bestGDLambda1, bestGDLambda2) = hpt3.Optimize(Fixed(transposeData), - Fixed(useCholesky), initialLambda1, initialLambda2); -@endcode - -@section hpt_class The HyperParameterTuner class - -The \c HyperParameterTuner class is very similar to the -\ref cv::KFoldCV "KFoldCV" and \ref cv::SimpleCV "SimpleCV" classes (see the -\ref cv "cross-validation tutorial" for more information on those two classes), but -there are a few important differences. - -First, the \c HyperParameterTuner accepts five different hyperparameters; only -the first three of these are required: - - - \c MLAlgorithm This is the algorithm to be used. - - \c Metric This is the performance measure to be used; see - @ref cvbasic_metrics for more information. - - \c CVType This is the type of cross-validation to be used for evaluating the - performance measure; this should be \ref cv::KFoldCV "KFoldCV" or - \ref cv::SimpleCV "SimpleCV". - - \c OptimizerType This is the type of optimizer to use; it can be - \c GridSearch or a gradient-based optimizer. - - \c MatType This is the type of data matrix to use. The default is - \c arma::mat. This only needs to be changed if you are specifically - using sparse data, or if you want to use a numeric type other than - \c double. - -The last two template parameters are automatically inferred by the -\c HyperParameterTuner and should not need to be manually specified, unless an -unconventional data type like \c arma::fmat is being used for data points. - -Typically, \ref cv::SimpleCV "SimpleCV" is a good choice for \c CVType because -it takes so much less time to compute than full \ref cv::KFoldCV "KFoldCV"; -however, the disadvantage is that \ref cv::SimpleCV "SimpleCV" might give a -somewhat more noisy estimate of the performance measure on unseen test data. - -The constructor for the \c HyperParameterTuner is called with exactly the same -arguments as the corresponding \c CVType that has been chosen. For more -information on that, please see the -@ref cvbasic_api "cross-validation constructor tutorial". As an example, if we -are using \ref cv::SimpleCV "SimpleCV" and wish to hold out 20\% of the dataset -as a validation set, we might construct a \c HyperParameterTuner like this: - -@code -// We will use LinearRegression as the MLAlgorithm, and MSE as the performance -// measure. Our dataset is 'dataset' and the responses are 'responses'. -HyperParameterTuner hpt(0.2, dataset, - responses); -@endcode - -Next, we must set up the hyperparameters to be optimized. If we are doing a -grid search with the \c ens::GridSearch optimizer (the -default), then we only need to pass a `std::vector` (for non-numeric -hyperparameters) or an `arma::vec` (for numeric hyperparameters) containing all -of the possible choices that we wish to search over. - -For instance, a set of numeric values might be chosen like this, for the -\c lambda parameter (of type \c double): - -@code -arma::vec lambdaSet = arma::vec("0.0 0.1 0.5 1.0"); -@endcode - -Similarly, a set of non-numeric values might be chosen like this, for the -\c intercept parameter: - -@code -std::vector interceptSet = { false, true }; -@endcode - -Once all of these are set up, the \c HyperParameterTuner::Optimize() method may -be called to find the best set of hyperparameters: - -@code -bool intercept; -double lambda; -std::tie(lambda, intercept) = hpt.Optimize(lambdaSet, interceptSet); -@endcode - -Alternately, the \c Fixed() method (detailed in the @ref hptfixed -"Fixed arguments" section) can be used to fix the values of some parameters. - -For continuous optimizers like -\c ens::GradientDescent, a range does not need to -be specified but instead only a single value. See the -\ref hptgradient "Gradient-Based Optimization" section for more details. - -@section hptfurther Further documentation - -For more information on the \c HyperParameterTuner class, see the -mlpack::hpt::HyperParameterTuner class documentation and the -@ref cv "cross-validation tutorial". - -*/ - -} // namespace hpt -} // namespace mlpack diff --git a/doc/guide/sample.hpp b/doc/guide/sample.hpp deleted file mode 100644 index dce8a8a154..0000000000 --- a/doc/guide/sample.hpp +++ /dev/null @@ -1,106 +0,0 @@ -/*! @page sample Simple Sample mlpack Programs - -@section sampleintro Introduction - -On this page, several simple mlpack examples are contained, in increasing order -of complexity. If you compile from the command-line, be sure that your compiler -is in C++11 mode. With modern gcc and clang, this should already be the -default. - -@note -The command-line programs like @c knn_main.cpp and @c -logistic_regression_main.cpp from the directory @c src/mlpack/methods/ cannot be -compiled easily by hand (the same is true for the individual tests in @c -src/mlpack/tests/); instead, those should be compiled with CMake, by running, -e.g., @c make @c mlpack_knn or @c make @c mlpack_test; see @ref build. However, -any program that uses mlpack (and is not a part of the library itself) can be -compiled easily with g++ or clang from the command line. - -@section covariance Covariance Computation - -A simple program to compute the covariance of a data matrix ("data.csv"), -assuming that the data is already centered, and save it to file. - -@code -// Includes all relevant components of mlpack. -#include - -// Convenience. -using namespace mlpack; - -int main() -{ - // First, load the data. - arma::mat data; - // Use data::Load() which transposes the matrix. - data::Load("data.csv", data, true); - - // Now compute the covariance. We assume that the data is already centered. - // Remember, because the matrix is column-major, the covariance operation is - // transposed. - arma::mat cov = data * trans(data) / data.n_cols; - - // Save the output. - data::Save("cov.csv", cov, true); -} -@endcode - -@section nn Nearest Neighbor - -This simple program uses the mlpack::neighbor::NeighborSearch object to find the -nearest neighbor of each point in a dataset using the L1 metric, and then print -the index of the neighbor and the distance of it to stdout. - -@code -#include -#include - -using namespace mlpack; -using namespace mlpack::neighbor; // NeighborSearch and NearestNeighborSort -using namespace mlpack::metric; // ManhattanDistance - -int main() -{ - // Load the data from data.csv (hard-coded). Use IO for simple command-line - // parameter handling. - arma::mat data; - data::Load("data.csv", data, true); - - // Use templates to specify that we want a NeighborSearch object which uses - // the Manhattan distance. - NeighborSearch nn(data); - - // Create the object we will store the nearest neighbors in. - arma::Mat neighbors; - arma::mat distances; // We need to store the distance too. - - // Compute the neighbors. - nn.Search(1, neighbors, distances); - - // Write each neighbor and distance using Log. - for (size_t i = 0; i < neighbors.n_elem; ++i) - { - std::cout << "Nearest neighbor of point " << i << " is point " - << neighbors[i] << " and the distance is " << distances[i] << ".\n"; - } -} -@endcode - -@section other Other examples - -For more complex examples, it is useful to refer to the main executables, found -in @c src/mlpack/methods/. A few are listed below. - - - methods/neighbor_search/knn_main.cpp - - methods/neighbor_search/kfn_main.cpp - - methods/emst/emst_main.cpp - - methods/radical/radical_main.cpp - - methods/nca/nca_main.cpp - - methods/naive_bayes/nbc_main.cpp - - methods/pca/pca_main.cpp - - methods/lars/lars_main.cpp - - methods/linear_regression/linear_regression_main.cpp - - methods/gmm/gmm_main.cpp - - methods/kmeans/kmeans_main.cpp - -*/ diff --git a/doc/guide/sample_ml_app.hpp b/doc/guide/sample_ml_app.hpp deleted file mode 100644 index 3de856c40b..0000000000 --- a/doc/guide/sample_ml_app.hpp +++ /dev/null @@ -1,203 +0,0 @@ -/** - * @file sample_ml_app.hpp - * @author German Lancioni - -@page sample_ml_app Sample C++ ML App for Windows - -@section sample_intro Introduction - -This tutorial will help you create a sample machine learning app using mlpack/C++. Although this app -does not cover all the mlpack capabilities, it will walkthrough several APIs to understand how -everything connects. This Windows sample app is created using Visual Studio, but you can easily -adapt it to a different platform by following the provided source code. - -@note Before starting, make sure you have built mlpack for Windows following this @ref build_windows "Windows guide" - -@section sample_create_project Creating the VS project - -- Open Visual Studio and create a new project (Windows Console Application) -- For this sample, the project is named “sample-ml-app” - -@section sample_project_config Project Configuration - -There are different ways in which you can configure your project to link with dependencies. This configuration -is for x64 Debug Mode. If you need Release Mode, please change the paths accordingly (assuming you have built -mlpack and dependencies in Release Mode). - -- Right click on the project and select Properties, select the x64 Debug profile -- Under C/C++ > General > Additional Include Directories add: -@code - - C:\mlpack\armadillo-9.800.3\include - - C:\mlpack\mlpack-3.4.2\src -@endcode -- Under Build Events > Post-Build Event > Command Line add: -@code - - xcopy /y "C:\mlpack\mlpack-3.4.2\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) -@endcode - -@note Recent versions of Visual Studio set "Conformance Mode" enabled by default. This causes some issues with -the armadillo library. If you encounter this issue, disable "Conformance Mode" under C/C++ > Language. - -@section sample_app_goal The app goal - -This app aims to exercise an end-to-end machine learning workflow. We will cover: - -- Loading and preparing a dataset -- Training (using Random Forest as example) -- Computing the training accuracy -- Cross-Validation using K-Fold -- Metrics gathering (accuracy, precision, recall, F1) -- Saving the trained model to disk -- Loading the model -- Classifying a new sample - -@section sample_headers_namespaces Headers and namespaces - -For this app, we will need to include the following headers (i.e. add into stdafx.h): - -@code -#include "mlpack/core.hpp" -#include "mlpack/methods/random_forest/random_forest.hpp" -#include "mlpack/methods/decision_tree/random_dimension_select.hpp" -#include "mlpack/core/cv/k_fold_cv.hpp" -#include "mlpack/core/cv/metrics/accuracy.hpp" -#include "mlpack/core/cv/metrics/precision.hpp" -#include "mlpack/core/cv/metrics/recall.hpp" -#include "mlpack/core/cv/metrics/F1.hpp" -@endcode - -Also, we will use the following namespaces: - -@code -using namespace arma; -using namespace mlpack; -using namespace mlpack::tree; -using namespace mlpack::cv; -@endcode - -@section sample_load_dataset Loading the dataset - -First step is about loading the dataset. Different dataset file formats are supported, but here -we load a CSV dataset, and we assume the labels don't require normalization. - -@note Make sure you update the path to your dataset file. For this sample, you can simply -copy "mlpack/tests/data/german.csv" and paste into a new "data" folder in your project directory. - -@code -mat dataset; -bool loaded = mlpack::data::Load("data/german.csv", dataset); -if (!loaded) - return -1; -@endcode - -Then we need to extract the labels from the last dimension of the dataset and remove the -labels from the training set: - -@code -Row labels; -labels = conv_to>::from(dataset.row(dataset.n_rows - 1)); -dataset.shed_row(dataset.n_rows - 1); -@endcode - -We now have our dataset ready for training. - -@section sample_training Training - -This app will use a Random Forest classifier. At first we define the classifier parameters and then -we create the classifier to train it. - -@code -const size_t numClasses = 2; -const size_t minimumLeafSize = 5; -const size_t numTrees = 10; - -RandomForest rf; - -rf = RandomForest(dataset, labels, - numClasses, numTrees, minimumLeafSize); -@endcode - -Now that the training is completed, we quickly compute the training accuracy: - -@code -Row predictions; -rf.Classify(dataset, predictions); -const size_t correct = arma::accu(predictions == labels); -cout << "\nTraining Accuracy: " << (double(correct) / double(labels.n_elem)); -@endcode - -@section sample_crossvalidation Cross-Validating - -Instead of training the Random Forest directly, we could also use K-fold cross-validation for training, -which will give us a measure of performance on a held-out test set. This can give us a better estimate -of how the model will perform when given new data. We also define which metric to use in order -to assess the quality of the trained model. - -@code -const size_t k = 10; -KFoldCV, Accuracy> cv(k, - dataset, labels, numClasses); -double cvAcc = cv.Evaluate(numTrees, minimumLeafSize); -cout << "\nKFoldCV Accuracy: " << cvAcc; -@endcode - -To compute other relevant metrics, such as Precision, Recall and F1: - -@code -double cvPrecision = Precision::Evaluate(rf, dataset, labels); -cout << "\nPrecision: " << cvPrecision; - -double cvRecall = Recall::Evaluate(rf, dataset, labels); -cout << "\nRecall: " << cvRecall; - -double cvF1 = F1::Evaluate(rf, dataset, labels); -cout << "\nF1: " << cvF1; -@endcode - -@section sample_save_model Saving the model - -Now that our model is trained and validated, we save it to a file so we can use it later. Here we save the -model that was trained using the entire dataset. Alternatively, we could extract the model from the cross-validation -stage by using \c cv.Model() - -@code -mlpack::data::Save("mymodel.xml", "model", rf, false); -@endcode - -We can also save the model in \c bin format ("mymodel.bin") which would result in a smaller file. - -@section sample_load_model Loading the model - -In a real-life application, you may want to load a previously trained model to classify new samples. -We load the model from a file using: - -@code -mlpack::data::Load("mymodel.xml", "model", rf); -@endcode - -@section sample_classify_sample Classifying a new sample - -Finally, the ultimate goal is to classify a new sample using the previously trained model. Since the -Random Forest classifier provides both predictions and probabilities, we obtain both. - -@code -// Create a test sample containing only one point. Because Armadillo is -// column-major, this matrix has one column (one point) and the number of rows -// is equal to the dimensionality of the point (23). -mat sample("2; 12; 2; 13; 1; 2; 2; 1; 3; 24; 3; 1; 1; 1; 1; 1; 0; 1; 0; 1;" - " 0; 0; 0"); -mat probabilities; -rf.Classify(sample, predictions, probabilities); -u64 result = predictions.at(0); -cout << "\nClassification result: " << result << " , Probabilities: " << - probabilities.at(0) << "/" << probabilities.at(1); -@endcode - -@section sample_app_conclussion Final thoughts - -Building real-life applications and services using machine learning can be challenging. Hopefully, this -tutorial provides a good starting point that covers the basic workflow you may need to follow while -developing it. You can take a look at the entire source code in the provided sample project located here: -"doc/examples/sample-ml-app". - -*/ diff --git a/doc/guide/timer.hpp b/doc/guide/timer.hpp deleted file mode 100644 index e9ba483d11..0000000000 --- a/doc/guide/timer.hpp +++ /dev/null @@ -1,66 +0,0 @@ -/*! @page timer mlpack Timers - -@section timerintro Introduction - -mlpack provides a simple timer interface for the timing of machine learning -methods. The results of any timers used during the program are displayed at -output by any command-line binding, when --verbose is given: - -@code -$ mlpack_knn -r dataset.csv -n neighbors_out.csv -d distances_out.csv -k 5 -v -<...> -[INFO ] Program timers: -[INFO ] computing_neighbors: 0.010650s -[INFO ] loading_data: 0.002567s -[INFO ] saving_data: 0.001115s -[INFO ] total_time: 0.149816s -[INFO ] tree_building: 0.000534s -@endcode - -@section usingtimer Timer API - -The mlpack::Timer class provides three simple methods: - -@code -void Timer::Start(const char* name); -void Timer::Stop(const char* name); -timeval Timer::Get(const char* name); -@endcode - -Each timer is given a name, and is referenced by that name. You can call \c -Timer::Start() and \c Timer::Stop() multiple times for a particular timer name, -and the result will be the sum of the runs of the timer. Note that \c -Timer::Stop() must be called before \c Timer::Start() is called again, -otherwise a std::runtime_error exception will be thrown. - -A \c "total_time" timer is run by default for each mlpack program. - -@section example Timer Example - -Below is a very simple example of timer usage in code. - -@code -#include -#include -#define BINDING_TYPE BINDING_TYPE_CLI -#include - -using namespace mlpack; - -void mlpackMain() -{ - // Start a timer. - Timer::Start("some_timer"); - - // Do some things. - DoSomeStuff(); - - // Stop the timer. - Timer::Stop("some_timer"); -} -@endcode - -If the --verbose flag was given to this executable, the time that -\c "some_timer" ran for would be printed at the end of the program's output. - -*/ diff --git a/doc/guide/version.hpp b/doc/guide/version.hpp deleted file mode 100644 index db3509ef8b..0000000000 --- a/doc/guide/version.hpp +++ /dev/null @@ -1,29 +0,0 @@ -/*! @page verinfo mlpack version information - -@section vercode mlpack versions in code - -mlpack provides a couple of convenience macros and functions to get the version -of mlpack. More information (and straightforward code) can be found in -src/mlpack/core/util/version.hpp. - -The following three macros provide major, minor, and patch versions of mlpack -(i.e. for mlpack-x.y.z, 'x' is the major version, 'y' is the minor version, and -'z' is the patch version): - -@code -MLPACK_VERSION_MAJOR -MLPACK_VERSION_MINOR -MLPACK_VERSION_PATCH -@endcode - -In addition, the function \c mlpack::util::GetVersion() returns the mlpack -version as a string (for instance, "mlpack 1.0.8"). - -@section verex mlpack executable versions - -Each mlpack executable supports the \c --version (or \c -V ) option, which will -print the version of mlpack used. If the version is not an official release but -instead from svn trunk, the version will be "mlpack trunk" (and may have a -revision number appended to "trunk"). - -*/ diff --git a/doc/policies/elemtype.hpp b/doc/policies/elemtype.hpp deleted file mode 100644 index af4f0afaa3..0000000000 --- a/doc/policies/elemtype.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/*! @page elem The ElemType policy in mlpack - -@section elem_overview Overview - -\b mlpack algorithms should be as generic as possible. Often this means -allowing arbitrary metrics or kernels to be used, but this also means allowing -any type of data point to be used. This means that \b mlpack classes should -support \c float, \c double, and other observation types. Some algorithms -support this through the use of a \c MatType template parameter; others will -have their own template parameter, \c ElemType. - -The \c ElemType template parameter can take any value that can be used by -Armadillo (or, specifically, classes like \c arma::Mat<> and others); this -encompasses the types - - - \c double - - \c float - - \c int - - \c unsigned int - - \c std::complex - - \c std::complex - -and other primitive numeric types. Note that Armadillo does not support some -integer types for functionality such as matrix decompositions or other more -advanced linear algebra. This means that when these integer types are used, -some algorithms may fail with Armadillo error messages indicating that those -types cannot be used. - -@section A note for developers - -If the class has a \c MatType template parameter, \c ElemType can be easily -defined as below: - -@code -typedef typename MatType::elem_type ElemType; -@endcode - -and otherwise a template parameter with the name \c ElemType can be used. It is -generally a good idea to expose the element type somehow for use by other -classes. - -*/ diff --git a/doc/policies/functiontype.hpp b/doc/policies/functiontype.hpp deleted file mode 100644 index aacaca0678..0000000000 --- a/doc/policies/functiontype.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/*! @page function The FunctionType policy in mlpack - -@section Overview - -To represent the various types of loss functions encountered in machine -learning problems, mlpack provides the \c FunctionType template parameter in -the optimizer interface. The various optimizers available in the core library -rely on this policy to gain the necessary information required by the optimizing -algorithm. - -The \c FunctionType template parameter required by the Optimizer class can have -additional requirements imposed on it, depending on the type of optimizer used. - -@section requirements Interface requirements - -The most basic requirements for the \c FunctionType parameter are the -implementations of two public member functions, with the following interface -and semantics - -@code -// Evaluate the loss function at the given coordinates. -double Evaluate(const arma::mat& coordinates); -@endcode - - -@code -// Evaluate the gradient at the given coordinates, where 'gradient' is an -// output parameter for the required gradient. -void Gradient(const arma::mat& coordinates, arma::mat& gradient); -@endcode - - -Optimizers like SGD and RMSProp require a \c DecomposableFunctionType having the -following requirements - -@code -// Return the number of functions. In a data-dependent function, this would -// return the number of points in the dataset. -size_t NumFunctions(); -@endcode - - -@code -// Evaluate the 'i' th loss function. For example, for a data-dependent -// function, Evaluate(coordinates, 0) should evaluate the loss function at the -// first point in the dataset. -double Evaluate(const arma::mat& coordinates, const size_t i); -@endcode - -@code -// Evaluate the gradient of the 'i' th loss function at the given coordinates, -// where 'gradient' is an output parameter for the required gradient. -void Gradient(const arma::mat& coordinates, const size_t i, arma::mat& gradient); -@endcode - - - -\c ParallelSGD optimizer requires a \c SparseFunctionType interface. -\c SparseFunctionType requires the gradient to be in a sparse matrix (\c -arma::sp_mat), as ParallelSGD, implemented with the HOGWILD! scheme of -unsynchronised updates, is expected to be relevant only in situations where the -individual gradients are sparse. So, the interface requires function with the -following signatures - -@code -// Return the number of functions. In a data-dependent function, this would -// return the number of points in the dataset. -size_t NumFunctions(); -@endcode - - -@code -// Evaluate the loss function at the given coordinates. -double Evaluate(const arma::mat& coordinates); -@endcode - - -@code -// Evaluate the (sparse) gradient of the 'i' th loss function at the given -// coordinates, where 'gradient' is an output parameter for the required -// gradient. -void Gradient(const arma::mat& coordinates, const size_t i, arma::sp_mat& gradient); -@endcode - - -The \c SCD optimizer requires a \c ResolvableFunctionType interface, to -calculate partial gradients with respect to individual features. The optimizer -requires the decision variable to be arranged in a particular fashion to allow -for disjoint updates. The features should be arranged columnwise in the decision -variable. For example, in \c SoftmaxRegressionFunction the decision variable has -size \c numClasses x \c featureSize (+ 1 if an intercept also needs to be fit). -Similarly, for \c LogisticRegression, the decision variable is a row vector, -with the number of columns determined by the dimensionality of the dataset. - -The interface expects the following member functions from the function class - -@code -// Return the number of features in the decision variable. -size_t NumFeatures(); -@endcode - -@code -// Evaluate the loss function at the given coordinates. -double Evaluate(const arma::mat& coordinates); -@endcode - -@code -// Evaluate the partial gradient of the loss function with respect to the 'j' th -// coordinate at the given coordinates, where 'gradient' is an output parameter -// for the required gradient. The 'gradient' matrix is supposed to be non-zero -// in the jth column, which contains the relevant partial gradient. -void PartialGradient(const arma::mat& coordinates, const size_t j, arma::sp_mat& gradient); -@endcode -*/ diff --git a/doc/policies/kernels.hpp b/doc/policies/kernels.hpp deleted file mode 100644 index 8a9bdad120..0000000000 --- a/doc/policies/kernels.hpp +++ /dev/null @@ -1,166 +0,0 @@ -/*! @page kernels The KernelType policy in mlpack - -@section kerneltoc Table of Contents - - - \ref kerneltype - - \ref kerneltraits - - \ref kernellist - -@section kerneltype Introduction to the KernelType policy - -`Kernel methods' make up a large class of machine learning techniques. Each of -these methods is characterized by its dependence on a \b kernel \b function. In -rough terms, a kernel function is a general notion of similarity between two -points, with its value large when objects are similar and its value small when -objects are dissimilar (note that this is not the only interpretation of what a -kernel is). - -A kernel (or `Mercer kernel') \f$\mathcal{K}(\cdot, \cdot)\f$ takes two objects -as input and returns some sort of similarity value. The specific details and -properties of kernels are outside the scope of this documentation; for a better -introduction to kernels and kernel methods, there are numerous better resources -available, including -Eric Kim's tutorial - -mlpack implements a number of kernel methods and, accordingly, each of these -methods allows arbitrary kernels to be used via the \c KernelType template -parameter. Like the \ref metrics "MetricType policy", the requirements are -quite simple: a class implementing the \c KernelType policy must have - - - an \c Evaluate() function - - a default constructor - -The signature of the \c Evaluate() function is straightforward: - -@code -template -double Evaluate(const VecTypeA& a, const VecTypeB& b); -@endcode - -The function takes two vector arguments, \c a and \c b, and returns a \c double -that is the evaluation of the kernel between the two arguments. So, for a -particular kernel \f$\mathcal{K}(\cdot, \cdot)\f$, the \c Evaluate() function -should return \f$\mathcal{K}(a, b)\f$. - -The arguments \c a and \c b, of types \c VecTypeA and \c VecTypeB, respectively, -will be an Armadillo-like vector type (usually \c arma::vec, \c arma::sp_vec, or -similar). In general it should be valid to assume that \c VecTypeA is a class -with the same API as \c arma::vec. - -Note that for kernels that do not hold any state, the \c Evaluate() method can -be marked as \c static. - -Overall, the \c KernelType template policy is quite simple (much like the -\ref metrics "MetricType policy"). Below is an example kernel class, which -outputs \c 1 if the vectors are close and \c 0 otherwise. - -@code -class ExampleKernel -{ - // Default constructor is required. - ExampleKernel() { } - - // The example kernel holds no state, so we can mark Evaluate() as static. - template - static double Evaluate(const VecTypeA& a, const VecTypeB& b) - { - // Get how far apart the vectors are (using the Euclidean distance). - const double distance = arma::norm(a - b); - - if (distance < 0.05) // Less than 0.05 distance is "close". - return 1; - else - return 0; - } -}; -@endcode - -Then, this kernel may be easily used inside of mlpack algorithms. For instance, -the code below runs kernel PCA (\c mlpack::kpca::KernelPCA) on a random dataset -using the \c ExampleKernel. The results are saved to a file called -\c results.csv. (Note that this is simply an example to demonstrate usage, and -this example kernel isn't actually likely to be useful in practice.) - -@code -#include -#include -#include "example_kernel.hpp" // Contains the ExampleKernel class. - -using namespace mlpack; -using namespace mlpack::kpca; -using namespace arma; - -int main() -{ - // Generate the random dataset; 10 dimensions, 5000 points. - mat dataset = randu(10, 5000); - - // Instantiate the KernelPCA object with the ExampleKernel kernel type. - KernelPCA kpca; - - // The dataset will be transformed using kernel PCA with the example kernel to - // contain only 2 dimensions. - kpca.Apply(dataset, 2); - - // Save the results to 'results.csv'. - data::Save(dataset, "results.csv"); -} -@endcode - -@section kerneltraits The KernelTraits trait class - -Some algorithms that use kernels can specialize if the kernel fulfills some -certain conditions. An example of a condition might be that the kernel is -shift-invariant or that the kernel is normalized. In the case of fast -max-kernel search (mlpack::fastmks::FastMKS), the computation can be accelerated -if the kernel is normalized. For this reason, the \c KernelTraits trait class -exists. This allows a kernel to specify via a \c const \c static \c bool when -these types of conditions are satisfied. **Note that a KernelTraits class -is not required,** but may be helpful. - -The \c KernelTraits trait class is a template class that takes a \c KernelType -as a parameter, and exposes \c const \c static \c bool values that depend on the -kernel. Setting these values is achieved by specialization. The code below -provides an example, specializing \c KernelTraits for the \c ExampleKernel from -earlier: - -@code -template<> -class KernelTraits -{ - public: - //! The example kernel is normalized (K(x, x) = 1 for all x). - const static bool IsNormalized = true; -}; -@endcode - -At this time, there is only one kernel trait that is used in mlpack code: - - - \c IsNormalized (defaults to \c false): if \f$ K(x, x) = 1 \; \forall x \f$, - then the kernel is normalized and this should be set to true. - -@section kernellist List of kernels and classes that use a \c KernelType - -mlpack comes with a number of pre-written kernels that satisfy the \c KernelType -policy: - - - mlpack::kernel::LinearKernel - - mlpack::kernel::ExampleKernel -- an example kernel with more documentation - - mlpack::kernel::GaussianKernel - - mlpack::kernel::HyperbolicTangentKernel - - mlpack::kernel::EpanechnikovKernel - - mlpack::kernel::CosineDistance - - mlpack::kernel::LaplacianKernel - - mlpack::kernel::PolynomialKernel - - mlpack::kernel::TriangularKernel - - mlpack::kernel::SphericalKernel - - mlpack::kernel::PSpectrumStringKernel -- operates on strings, not vectors - -These kernels (or a custom kernel) may be used in a variety of mlpack methods: - - - mlpack::kpca::KernelPCA - kernel principal components analysis - - mlpack::fastmks::FastMKS - fast max-kernel search - - mlpack::kernel::NystroemMethod - the Nystroem method for sampling - - mlpack::metric::IPMetric - a metric built on a kernel - -*/ diff --git a/doc/guide/r_quickstart.hpp b/doc/quickstart/R.md similarity index 64% rename from doc/guide/r_quickstart.hpp rename to doc/quickstart/R.md index 36a8059067..2a9cee4d41 100644 --- a/doc/guide/r_quickstart.hpp +++ b/doc/quickstart/R.md @@ -1,45 +1,35 @@ -/** - * @file r_quickstart.hpp - * @author Yashwant Singh Parihar - -@page r_quickstart mlpack in R quickstart guide - -@section r_quickstart_intro Introduction +# mlpack in R quickstart guide This page describes how you can quickly get started using mlpack from R and gives a few examples of usage, and pointers to deeper documentation. -This quickstart guide is also available for @ref python_quickstart "Python", -@ref cli_quickstart "the command-line", @ref julia_quickstart "Julia" and -@ref go_quickstart "Go". +This quickstart guide is also available for [C++](cpp.md), [Python](python.md), +[Julia](julia.md), [the command line](cli.md), and [Go](go.md). -@section r_quickstart_install Installing mlpack binary package +## Installing mlpack Installing the mlpack bindings for R is straightforward; you can just use CRAN: -@code{.R} +```r install.packages('mlpack') -@endcode - -@section r_quickstart_source_install Installing mlpack package from source +``` Building the R bindings from scratch is a little more in-depth, though. For -information on that, follow the instructions on the @ref build page, and be sure -to specify @c -DBUILD_R_BINDINGS=ON to CMake; you may need to also set the -location of the R program with @c -DR_EXECUTABLE=/path/to/R. +information on that, follow the instructions in the +[main README](../../README.md). -@section r_quickstart_example Simple mlpack quickstart example +## Simple mlpack quickstart example As a really simple example of how to use mlpack from R, let's do some -simple classification on a subset of the standard machine learning @c covertype +simple classification on a subset of the standard machine learning `covertype` dataset. We'll first split the dataset into a training set and a testing set, then we'll train an mlpack random forest on the training data, and finally we'll print the accuracy of the random forest on the test dataset. You can copy-paste this code directly into R to run it. -@code{.R} +```r if(!requireNamespace("data.table", quietly = TRUE)) { install.packages("data.table") } suppressMessages({ library("mlpack") @@ -79,38 +69,26 @@ output <- random_forest(input_model = rf_model, correct <- sum(output$predictions == prepdata$test_labels) cat(correct, "out of", length(prepdata$test_labels), "test points correct", correct / length(prepdata$test_labels) * 100.0, "%\n") -@endcode +``` We can see that we achieve reasonably good accuracy on the test dataset (80%+); -if we use the full @c covertype.csv.gz, the accuracy should increase +if we use the full `covertype.csv.gz`, the accuracy should increase significantly (but training will take longer). It's easy to modify the code above to do more complex things, or to use different mlpack learners, or to interface with other machine learning toolkits. -@section r_quickstart_whatelse What else does mlpack implement? - -The example above has only shown a little bit of the functionality of mlpack. -Lots of other commands are available with different functionality. A full list -of each of these commands and full documentation can be found on the following -page: - - - r documentation - -For more information on what mlpack does, see https://www.mlpack.org/. -Next, let's go through another example for providing movie recommendations with -mlpack. - -@section r_quickstart_movierecs Using mlpack for movie recommendations +## Using mlpack for movie recommendations In this example, we'll train a collaborative filtering model using mlpack's -cf() method. We'll train this on the MovieLens dataset from -https://grouplens.org/datasets/movielens/, and then we'll use the model that we -train to give recommendations. +[`cf()`](https://www.mlpack.org/doc/stable/r_documentation.html#cf) method. +We'll train this on the +[MovieLens dataset](https://grouplens.org/datasets/movielens/), and then we'll +use the model that we train to give recommendations. You can copy-paste this code directly into R to run it. -@code{.R} +```r if(!requireNamespace("data.table", quietly = TRUE)) { install.packages("data.table") } suppressMessages({ library("mlpack") @@ -148,12 +126,12 @@ cat("Recommendations for user 1:\n") for (i in 1:10) { cat(" ", i, ":", as.character(movies[output$output[i], 3]), "\n") } -@endcode +``` Here is some example output, showing that user 1 seems to have good taste in movies: -@code{.unparsed} +``` Recommendations for user 1: 0: Casablanca (1942) 1: Pan's Labyrinth (Laberinto del fauno, El) (2006) @@ -165,29 +143,20 @@ Recommendations for user 1: 7: Out for Justice (1991) 8: Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1964) 9: Schindler's List (1993) -@endcode +``` -@section r_quickstart_nextsteps Next steps with mlpack +## Next steps with mlpack After working through this overview to `mlpack`'s R package, we hope you are -inspired to use `mlpack`' in your data science workflow. We recommend as part -of your next steps to look at more documentation for the R mlpack bindings: +inspired to use `mlpack`' in your data science workflow. However, the two +examples above have only shown a little bit of the functionality of mlpack. +Lots of other functions are available with different functionality. A full list +of each of these functions and full documentation can be found on the following +page: - - R mlpack - binding documentation + - [R documentation](https://www.mlpack.org/doc/stable/r_documentation.html) Also, mlpack is much more flexible from C++ and allows much greater functionality. So, more complicated tasks are possible if you are willing to -write C++ (or perhaps Rcpp). To get started learning about mlpack in C++, the -following resources might be helpful: - - - mlpack - C++ tutorials - - mlpack - build and installation guide - - Simple - sample C++ mlpack programs - - mlpack - Doxygen documentation homepage - - */ +write C++ (or perhaps Rcpp). To get started learning about mlpack in C++, a +good starting point is the [C++ quickstart guide](cpp.md). diff --git a/doc/guide/cli_quickstart.hpp b/doc/quickstart/cli.md similarity index 68% rename from doc/guide/cli_quickstart.hpp rename to doc/quickstart/cli.md index 447887935c..c0f5288ee7 100644 --- a/doc/guide/cli_quickstart.hpp +++ b/doc/quickstart/cli.md @@ -1,58 +1,51 @@ -/** - * @file cli_quickstart.hpp - * @author Ryan Curtin - -@page cli_quickstart mlpack command-line quickstart guide - -@section cli_quickstart_intro Introduction +# mlpack command-line quickstart guide This page describes how you can quickly get started using mlpack from the command-line and gives a few examples of usage, and pointers to deeper documentation. -This quickstart guide is also available for @ref python_quickstart "Python", -@ref r_quickstart "R", @ref julia_quickstart "Julia" and -@ref go_quickstart "Go". +This quickstart guide is also available for [C++](cpp.md), [Python](python.md), +[R](R.md), [Julia](julia.md), and [Go](go.md). -@section cli_quickstart_install Installing mlpack +## Installing mlpack -Installing the mlpack is straightforward and can be done with your system's -package manager. +Installing mlpack is straightforward and can be done with your system's package +manager. For instance, for Ubuntu or Debian the command is simply -For instance, for Ubuntu or Debian the command is simply - -@code{.sh} +```sh sudo apt-get install mlpack-bin -@endcode +``` On Fedora or Red Hat: -@code{.sh} +```sh sudo dnf install mlpack -@endcode +``` If you use a different distribution, mlpack may be packaged under a different name. And if it is not packaged, you can use a Docker image from Dockerhub: -@code{.sh} +```sh docker run -it mlpack/mlpack /bin/bash -@endcode +``` -This Docker image has mlpack already built and installed. +This Docker image has mlpack's command-line bindings already built and +installed. -If you prefer to build mlpack from scratch, see @ref build. +If you prefer to build mlpack from scratch, see the +[main README](../../README.md). -@section cli_quickstart_example Simple mlpack quickstart example +## Simple quickstart example As a really simple example of how to use mlpack from the command-line, let's do -some simple classification on a subset of the standard machine learning @c -covertype dataset. We'll first split the dataset into a training set and a +some simple classification on a subset of the standard machine learning +`covertype` dataset. We'll first split the dataset into a training set and a testing set, then we'll train an mlpack random forest on the training data, and finally we'll print the accuracy of the random forest on the test dataset. You can copy-paste this code directly into your shell to run it. -@code{.sh} +```sh # Get the dataset and unpack it. wget https://www.mlpack.org/datasets/covertype-small.data.csv.gz wget https://www.mlpack.org/datasets/covertype-small.labels.csv.gz @@ -89,42 +82,30 @@ mlpack_random_forest \ --test_labels_file covertype-small.test.labels.csv \ --predictions_file predictions.csv \ --verbose -@endcode +``` We can see by looking at the output that we achieve reasonably good accuracy on -the test dataset (80%+). The file @c predictions.csv could also be used by +the test dataset (80%+). The file `predictions.csv` could also be used by other tools; for instance, we can easily calculate the number of points that were predicted incorrectly: -@code{.sh} +```sh $ diff -U 0 predictions.csv covertype-small.test.labels.csv | grep '^@@' | wc -l -@endcode +``` It's easy to modify the code above to do more complex things, or to use different mlpack learners, or to interface with other machine learning toolkits. -@section cli_quickstart_whatelse What else does mlpack implement? - -The example above has only shown a little bit of the functionality of mlpack. -Lots of other commands are available with different functionality. A full list -of commands and full documentation for each can be found on the following page: - - - CLI documentation - -For more information on what mlpack does, see https://www.mlpack.org/. Next, -let's go through another example for providing movie recommendations with -mlpack. - -@section cli_quickstart_movierecs Using mlpack for movie recommendations +## Using mlpack for movie recommendations In this example, we'll train a collaborative filtering model using mlpack's -@c mlpack_cf program. We'll train this on the MovieLens dataset from -https://grouplens.org/datasets/movielens/, and then we'll use the model that we -train to give recommendations. +`mlpack_cf` program. We'll train this on the +[MovieLens dataset](https://grouplens.org/datasets/movielens/), and then we'll +use the model that we train to give recommendations. You can copy-paste this code directly into the command line to run it. -@code{.sh} +```sh wget https://www.mlpack.org/datasets/ml-20m/ratings-only.csv.gz wget https://www.mlpack.org/datasets/ml-20m/movies.csv.gz gunzip ratings-only.csv.gz @@ -165,12 +146,12 @@ for i in `seq 1 10`; do sed 's/^[^,]*,[^,]*,//' | \ sed 's/\(.*\),.*$/\1/' | sed 's/"//g'; done -@endcode +``` Here is some example output, showing that user 1 seems to have good taste in movies: -@code{.unparsed} +``` Recommendations for user 1: Casablanca (1942) Pan's Labyrinth (Laberinto del fauno, El) (2006) @@ -182,30 +163,18 @@ Dark Knight, The (2008) Out for Justice (1991) Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1964) Schindler's List (1993) -@endcode - -@section cli_quickstart_nextsteps Next steps with mlpack +``` +## Next steps with mlpack Now that you have done some simple work with mlpack, you have seen how it can -easily plug into a data science production workflow for the command line. A -great thing to do next would be to look at more documentation for the mlpack -command-line programs: +easily plug into a data science production workflow for the command line. But +these two examples have only shown a little bit of the functionality of mlpack. +Lots of other commands are available with different functionality. A full list +of commands and full documentation for each can be found on the following page: - - mlpack - command-line program documentation + - [CLI program documentation](https://www.mlpack.org/doc/stable/cli_documentation.html) Also, mlpack is much more flexible from C++ and allows much greater functionality. So, more complicated tasks are possible if you are willing to -write C++. To get started learning about mlpack in C++, the following resources -might be helpful: - - - mlpack - C++ tutorials - - mlpack - build and installation guide - - Simple - sample C++ mlpack programs - - mlpack - Doxygen documentation homepage - - */ +write C++. To get started learning about mlpack in C++, the +[C++ quickstart](cpp.md) is a good place to start. diff --git a/doc/quickstart/cpp.md b/doc/quickstart/cpp.md new file mode 100644 index 0000000000..5cf25ff32e --- /dev/null +++ b/doc/quickstart/cpp.md @@ -0,0 +1,276 @@ +# mlpack in C++ quickstart + +This page describes how you can quickly get started using mlpack in C++ and +gives a few examples of usage, and pointers to deeper documentation. + +Keep in mind that mlpack also has interfaces to other languages, and quickstart +guides for those other languages are available too. If that is what you are +looking for, see the quickstarts for [Python](python.md), +[the command line](cli.md), [Julia](julia.md), [R](R.md), or [Go](go.md). + +## Installing mlpack + +To use mlpack in C++, you only need the header files associated with the +libraries, and the dependencies Armadillo and ensmallen (detailed in the +[main README](../../README.md)). The headers may already be pre-packaged for +your distribution; for instance, for Ubuntu and Debian you can simply run the +command + +```sh +sudo apt-get install mlpack-dev +``` + +and on Fedora or Red Hat: + +```sh +sudo dnf install mlpack +``` + +If you run a different distribution, mlpack may be packaged under a different +name. And if it is not packaged, you can use a Docker image from Dockerhub: + +```sh +docker run -it mlpack/mlpack /bin/bash +``` + +This Docker image has mlpack headers already installed. + +If you prefer to build mlpack from scratch, see the +[main README](../../README.md). + +## Installing mlpack from vcpkg + +The mlpack port in vcpkg is kept up to date by Microsoft team members and community contributors. The url of vcpkg is: https://github.com/Microsoft/vcpkg . You can download and install mlpack using the vcpkg dependency manager: + +```shell +git clone https://github.com/Microsoft/vcpkg.git +cd vcpkg +./bootstrap-vcpkg.sh # ./bootstrap-vcpkg.bat for Windows +./vcpkg integrate install +./vcpkg install mlpack +``` + +If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + +## Simple quickstart example + +As a really simple example of how to use mlpack in C++, let's do some simple +classification on a subset of the standard machine learning `covertype` dataset. +We'll first split the dataset into a training set and a test set, then we'll +train an mlpack random forest on the training data, and finally we'll print the +accuracy of the random forest on the test dataset. + +The first step is to download the covertype dataset onto your system so that it +is available for the program. A shell command below is given to do this: + +```sh +# Get the dataset and unpack it. +wget https://www.mlpack.org/datasets/covertype-small.data.csv.gz +wget https://www.mlpack.org/datasets/covertype-small.labels.csv.gz +gunzip covertype-small.data.csv.gz covertype-small.labels.csv.gz +``` + +With that in place, let's write a C++ program to split the data and perform the +classification: + +```c++ +#include + +using namespace arma; +using namespace mlpack; +using namespace mlpack::tree; +using namespace std; + +int main() +{ + // Load the datasets. + mat dataset; + Row labels; + if (!data::Load("covertype-small.data.csv", dataset)) + throw std::runtime_error("Could not read covertype-small.data.csv!"); + if (!data::Load("covertype-small.labels.csv", labels)) + throw std::runtime_error("Could not read covertype-small.labels.csv!"); + + // Labels are 1-7, but we want 0-6 (we are 0-indexed in C++). + labels -= 1; + + // Now split the dataset into a training set and test set, using 30% of the + // dataset for the test set. + mat trainDataset, testDataset; + Row trainLabels, testLabels; + data::Split(dataset, labels, trainDataset, testDataset, trainLabels, + testLabels, 0.3); + + // Create the RandomForest object and train it on the training data. + RandomForest r(trainDataset, + trainLabels, + 7 /* number of classes */, + 10 /* number of trees */, + 3 /* minimum leaf size */); + + // Compute and print the training error. + Row trainPredictions; + r.Classify(trainDataset, trainPredictions); + const double trainError = + arma::accu(trainPredictions != trainLabels) * 100.0 / trainLabels.n_elem; + cout << "Training error: " << trainError << "%." << endl; + + // Now compute predictions on the test points. + Row testPredictions; + r.Classify(testDataset, testPredictions); + const double testError = + arma::accu(testPredictions != testLabels) * 100.0 / testLabels.n_elem; + cout << "Test error: " << testError << "%." << endl; +} +``` + +Now, you can compile the program with your favorite C++ compiler; here's an +example command that uses `g++`, and assumes the file above is saved as +`cpp_quickstart_1.cpp`. + +```sh +g++ -O3 -std=c++14 -o cpp_quickstart_1 cpp_quickstart_1.cpp -larmadillo -fopenmp +``` + +Then, you can run the program easily: + +```sh +./cpp_quickstart_1 +``` + +We can see by looking at the output that we achieve reasonably good accuracy on +the test dataset (80%+): + +``` +Training error: 19.4329%. +Test error: 24.17%. +``` + +It's easy to modify the code above to do more complex things, or to use +different mlpack learners, or to interface with other machine learning toolkits. + +## Using mlpack for movie recommendations + +In this example, we'll train a collaborative filtering model using mlpack's `CF` +class. We'll train this on this +[MovieLens dataset](https://grouplens.org/datasets/movielens/), and then we'll +use the model that we train to give recommendations. + +First, download the MovieLens dataset: + +```sh +wget https://www.mlpack.org/datasets/ml-20m/ratings-only.csv.gz +wget https://www.mlpack.org/datasets/ml-20m/movies.csv.gz +gunzip ratings-only.csv.gz movies.csv.gz +``` + +Next, we can use the following C++ code: + +```cpp +#include + +using namespace arma; +using namespace mlpack; +using namespace mlpack::cf; +using namespace std; + +int main() +{ + // Load the ratings. + mat ratings; + if (!data::Load("ratings-only.csv", ratings)) + throw std::runtime_error("Could not load ratings-only.csv!"); + // Now, load the names of the movies as a single-feature categorical dataset. + // We can use `moviesInfo.UnmapString(i, 0)` to get the i'th string. + data::DatasetInfo moviesInfo; + mat movies; // This will be unneeded. + if (!data::Load("movies.csv", movies, moviesInfo)) + throw std::runtime_error("Could not load movies.csv!"); + + // Split the ratings into a training set and a test set, using 10% of the + // dataset for the test set. + mat trainRatings, testRatings; + data::Split(ratings, trainRatings, testRatings, 0.1); + + // Train the CF model using RegularizedSVD as the decomposition algorithm. + // Here we use a rank of 10 for the decomposition. + CFType cf( + trainRatings, + RegSVDPolicy(), + 5, /* number of users to use for similarity computations */ + 10 /* rank of decomposition */); + + // Now compute the RMSE for the test set user and item combinations. To do + // this we must assemble the list of users and items. + Mat combinations(2, testRatings.n_cols); + for (size_t i = 0; i < testRatings.n_cols; ++i) + { + combinations(0, i) = size_t(testRatings(0, i)); // (user) + combinations(1, i) = size_t(testRatings(1, i)); // (item) + } + vec predictions; + cf.Predict(combinations, predictions); + const double rmse = norm(predictions - testRatings.row(2).t(), 2) / + sqrt((double) testRatings.n_cols); + std::cout << "RMSE of trained model is " << rmse << "." << endl; + + // Compute the top 10 movies for user 1. + Col users = { 1 }; + Mat recommendations; + cf.GetRecommendations(10, recommendations, users); + + // Now print each movie. + cout << "Recommendations for user 1:" << endl; + for (size_t i = 0; i < recommendations.n_elem; ++i) + { + cout << " " << (i + 1) << ". " + << moviesInfo.UnmapString(recommendations[i], 2) << "." << endl; + } +} +``` + +This can be compiled the same way as before, assuming the code is saved as +`cpp_quickstart_2.cpp`: + +```sh +g++ -O3 -std=c++14 -o cpp_quickstart_2 cpp_quickstart_2.cpp -fopenmp -larmadillo +``` + +And then it can be easily run: + +``` +./cpp_quickstart_2 +``` + +Here is some example output, showing that user 1 seems to have good taste in +movies: + +``` +RMSE of trained model is 0.795323. +Recommendations for user 1: + 1: Casablanca (1942) + 2: Pan's Labyrinth (Laberinto del fauno, El) (2006) + 3: Godfather, The (1972) + 4: Answer This! (2010) + 5: Life Is Beautiful (La Vita è bella) (1997) + 6: Adventures of Tintin, The (2011) + 7: Dark Knight, The (2008) + 8: Out for Justice (1991) + 9: Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1964) + 10: Schindler's List (1993) +``` + +## Next steps with mlpack + +Now that you have done some simple work with mlpack, you have seen how it can +easily plug into a data science production workflow in C++. But these two +examples have only shown a little bit of the functionality of mlpack. Lots of +other functionality is available. + +Some of this functionality is demonstrated in the +[examples repository](https://github.com/mlpack/examples). + +A full list of all classes and functions that mlpack implements can be found in +the +[Doxygen documentation](https://www.mlpack.org/doc/stable/doxygen/index.html), +or simply by browsing the well-commented source code. diff --git a/doc/guide/go_quickstart.hpp b/doc/quickstart/go.md similarity index 67% rename from doc/guide/go_quickstart.hpp rename to doc/quickstart/go.md index 522d90541e..7ca79a2a79 100644 --- a/doc/guide/go_quickstart.hpp +++ b/doc/quickstart/go.md @@ -1,43 +1,36 @@ -/** - * @file go_quickstart.hpp - * @author Yashwant Singh Parihar - -@page go_quickstart mlpack in Go quickstart guide - -@section go_quickstart_intro Introduction +# mlpack in Go quickstart guide This page describes how you can quickly get started using mlpack from Go and gives a few examples of usage, and pointers to deeper documentation. -This quickstart guide is also available for @ref python_quickstart "Python", -@ref cli_quickstart "the command-line", @ref julia_quickstart "Julia" and -@ref r_quickstart "R". +This quickstart guide is also available for [C++](cpp.md), [Python](python.md), +[Julia](julia.md), [the command line](cli.md), and [R](R.md). -@section go_quickstart_install Installing mlpack +## Installing mlpack Installing the mlpack bindings for Go is somewhat time-consuming as the library must be built; you can run the following code: -@code{.sh} +```sh go get -u -d mlpack.org/v1/mlpack cd ${GOPATH}/src/mlpack.org/v1/mlpack make install -@endcode - +``` Building the Go bindings from scratch is a little more in-depth, though. For -information on that, follow the instructions on the @ref build page, and be sure -to specify @c -DBUILD_GO_BINDINGS=ON to CMake; +information on that, follow the instructions in the +[main README](../../README.md). -@section go_quickstart_example Simple mlpack quickstart example +## Simple mlpack quickstart example As a really simple example of how to use mlpack from Go, let's do some -simple classification on a subset of the standard machine learning @c covertype +simple classification on a subset of the standard machine learning `covertype` dataset. We'll first split the dataset into a training set and a testing set, then we'll train an mlpack random forest on the training data, and finally we'll print the accuracy of the random forest on the test dataset. You can copy-paste this code directly into main.go to run it. -@code{.go} + +```go package main import ( @@ -95,41 +88,26 @@ func main() { fmt.Print(sum, " correct out of ", rows, " (", (float64(sum) / float64(rows)) * 100, "%).\n") } -@endcode +``` We can see that we achieve reasonably good accuracy on the test dataset (80%+); -if we use the full @c covertype.csv.gz, the accuracy should increase +if we use the full `covertype.csv.gz`, the accuracy should increase significantly (but training will take longer). It's easy to modify the code above to do more complex things, or to use different mlpack learners, or to interface with other machine learning toolkits. -@section go_quickstart_whatelse What else does mlpack implement? - -The example above has only shown a little bit of the functionality of mlpack. -Lots of other commands are available with different functionality. A full list -of each of these commands and full documentation can be found on the following -page: - - - Go documentation - -You can also use the GoDoc to explore the @c mlpack module and its -functions; every function comes with comprehensive documentation. - -For more information on what mlpack does, see https://www.mlpack.org/. -Next, let's go through another example for providing movie recommendations with -mlpack. - -@section go_quickstart_movierecs Using mlpack for movie recommendations +## Using mlpack for movie recommendations In this example, we'll train a collaborative filtering model using mlpack's -Cf() method. We'll train this on the MovieLens dataset from -https://grouplens.org/datasets/movielens/, and then we'll use the model that we -train to give recommendations. +[`cf()`](https://www.mlpack.org/doc/stable/go_documentation.html#cf) method. +We'll train this on the +[MovieLens dataset](https://grouplens.org/datasets/movielens/), and then we'll +use the model that we train to give recommendations. You can copy-paste this code directly into main.go to run it. -@code{.go} +```go package main import ( @@ -185,12 +163,12 @@ func main() { fmt.Println(i, ":", movies[int(output.At(0 , i))]) } } -@endcode +``` Here is some example output, showing that user 1 seems to have good taste in movies: -@code{.unparsed} +``` Recommendations for user 1: 0: Casablanca (1942) 1: Pan's Labyrinth (Laberinto del fauno, El) (2006) @@ -202,29 +180,22 @@ Recommendations for user 1: 7: Out for Justice (1991) 8: Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1964) 9: Schindler's List (1993) -@endcode +``` -@section go_quickstart_nextsteps Next steps with mlpack +## Next steps with mlpack Now that you have done some simple work with mlpack, you have seen how it can -easily plug into a data science workflow in Go. A great thing to do next -would be to look at more documentation for the Go mlpack bindings: +easily plug into a data science workflow in Go. But the two examples above have +only shown a little bit of the functionality of mlpack. Lots of other methods +are available with different functionality. A full list of each of these +methods and full documentation can be found on the following page: - - Go mlpack - binding documentation + - [mlpack Go binding documentation](https://www.mlpack.org/doc/stable/go_documentation.html) + +You can also use GoDoc to explore the `mlpack` module and its functions; every +function comes with comprehensive documentation. Also, mlpack is much more flexible from C++ and allows much greater functionality. So, more complicated tasks are possible if you are willing to -write C++. To get started learning about mlpack in C++, the following resources -might be helpful: - - - mlpack - C++ tutorials - - mlpack - build and installation guide - - Simple - sample C++ mlpack programs - - mlpack - Doxygen documentation homepage - - */ +write C++. To get started learning about mlpack in C++, the +[C++ quickstart](cpp.md) is a good resource to visit next. diff --git a/doc/guide/julia_quickstart.hpp b/doc/quickstart/julia.md similarity index 63% rename from doc/guide/julia_quickstart.hpp rename to doc/quickstart/julia.md index ac7203f52c..3000bed21e 100644 --- a/doc/guide/julia_quickstart.hpp +++ b/doc/quickstart/julia.md @@ -1,37 +1,29 @@ -/** - * @file julia_quickstart.hpp - * @author Ryan Curtin - -@page julia_quickstart mlpack in Julia quickstart guide - -@section julia_quickstart_intro Introduction +# mlpack in Julia quickstart guide This page describes how you can quickly get started using mlpack from Julia and gives a few examples of usage, and pointers to deeper documentation. -This quickstart guide is also available for @ref python_quickstart "Python", -@ref cli_quickstart "the command-line", @ref go_quickstart "Go" and -@ref r_quickstart "R". +This quickstart guide is also available for [C++](cpp.md), [Python](python.md), +[the command line](cli.md), [R](R.md), and [Go](go.md). -@section julia_quickstart_install Installing mlpack +## Installing mlpack Installing the mlpack bindings for Julia is straightforward; you can just use -@c Pkg: +`Pkg`: -@code{.julia} +```julia using Pkg Pkg.add("mlpack") -@endcode +``` Building the Julia bindings from scratch is a little more in-depth, though. For -information on that, follow the instructions on the @ref build page, and be sure -to specify @c -DBUILD_JULIA_BINDINGS=ON to CMake; you may need to also set the -location of the Julia program with @c -DJULIA_EXECUTABLE=/path/to/julia. +information on that, follow the instructions in the +[main README](../../README.md). -@section julia_quickstart_example Simple mlpack quickstart example +## Simple quickstart example As a really simple example of how to use mlpack from Julia, let's do some -simple classification on a subset of the standard machine learning @c covertype +simple classification on a subset of the standard machine learning `covertype` dataset. We'll first split the dataset into a training set and a testing set, then we'll train an mlpack random forest on the training data, and finally we'll print the accuracy of the random forest on the test dataset. @@ -40,7 +32,7 @@ You can copy-paste this code directly into Julia to run it. You may need to add some extra packages with, e.g., `using Pkg; Pkg.add("CSV"); Pkg.add("DataFrames"); Pkg.add("Libz")`. -@code{.julia} +```julia using CSV using DataFrames using Libz @@ -77,41 +69,26 @@ _, predictions, _ = mlpack.random_forest(input_model=rf_model, correct = sum(predictions .== test_labels) print("$(correct) out of $(length(test_labels)) test points correct " * "($(correct / length(test_labels) * 100.0)%).\n") -@endcode +``` We can see that we achieve reasonably good accuracy on the test dataset (80%+); -if we use the full @c covertype.csv.gz, the accuracy should increase +if we use the full `covertype.csv.gz`, the accuracy should increase significantly (but training will take longer). It's easy to modify the code above to do more complex things, or to use different mlpack learners, or to interface with other machine learning toolkits. -@section julia_quickstart_whatelse What else does mlpack implement? - -The example above has only shown a little bit of the functionality of mlpack. -Lots of other commands are available with different functionality. A full list -of each of these commands and full documentation can be found on the following -page: - - - Julia documentation - -You can also use the Julia REPL to explore the @c mlpack module and its -functions; every function comes with comprehensive documentation. - -For more information on what mlpack does, see https://www.mlpack.org/. -Next, let's go through another example for providing movie recommendations with -mlpack. - -@section julia_quickstart_movierecs Using mlpack for movie recommendations +## Using mlpack for movie recommendations In this example, we'll train a collaborative filtering model using mlpack's -cf() method. We'll train this on the MovieLens dataset from -https://grouplens.org/datasets/movielens/, and then we'll use the model that we -train to give recommendations. +[`cf()`](https://www.mlpack.org/doc/stable/julia_documentation.html#cf) method. +We'll train this on the +[MovieLens dataset](https://grouplens.org/datasets/movielens/), and then we'll +use the model that we train to give recommendations. You can copy-paste this code directly into Julia to run it. -@code{.julia} +```julia using CSV using mlpack using Libz @@ -147,12 +124,12 @@ print("Recommendations for user 1:\n") for i in 1:10 print(" $(i): $(movies[output[i], :][3])\n") end -@endcode +``` Here is some example output, showing that user 1 seems to have good taste in movies: -@code{.unparsed} +``` Recommendations for user 1: 0: Casablanca (1942) 1: Pan's Labyrinth (Laberinto del fauno, El) (2006) @@ -164,29 +141,22 @@ Recommendations for user 1: 7: Out for Justice (1991) 8: Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1964) 9: Schindler's List (1993) -@endcode +``` -@section julia_quickstart_nextsteps Next steps with mlpack +## Next steps with mlpack Now that you have done some simple work with mlpack, you have seen how it can -easily plug into a data science workflow in Julia. A great thing to do next -would be to look at more documentation for the Julia mlpack bindings: +easily plug into a data science workflow in Julia. But the two examples above +have only shown a little bit of the functionality of mlpack. Lots of other +functions are available with different functionality. A full list of each of +these commands and full documentation can be found on the following page: - - Julia mlpack - binding documentation + - [Julia documentation](https://www.mlpack.org/doc/stable/julia_documentation.html) + +You can also use the Julia REPL to explore the `mlpack` module and its +functions; every function comes with comprehensive documentation. Also, mlpack is much more flexible from C++ and allows much greater functionality. So, more complicated tasks are possible if you are willing to write C++ (or perhaps CxxWrap.jl). To get started learning about mlpack in C++, -the following resources might be helpful: - - - mlpack - C++ tutorials - - mlpack - build and installation guide - - Simple - sample C++ mlpack programs - - mlpack - Doxygen documentation homepage - - */ +the [C++ quickstart](cpp.md) would be a good place to start. diff --git a/doc/guide/python_quickstart.hpp b/doc/quickstart/python.md similarity index 60% rename from doc/guide/python_quickstart.hpp rename to doc/quickstart/python.md index faf2fe2638..e3f66938b4 100644 --- a/doc/guide/python_quickstart.hpp +++ b/doc/quickstart/python.md @@ -1,69 +1,45 @@ -/** - * @file python_quickstart.hpp - * @author Ryan Curtin - -@page python_quickstart mlpack in Python quickstart guide - -@section python_quickstart_intro Introduction +# mlpack in Python quickstart guide This page describes how you can quickly get started using mlpack from Python and gives a few examples of usage, and pointers to deeper documentation. -This quickstart guide is also available for -@ref cli_quickstart "the command-line" and @ref julia_quickstart "Julia". +This quickstart guide is also available for [C++](cpp.md), +[the command line](cli.md), [Julia](julia.md), [R](R.md), and [Go](go.md). -@section python_quickstart_install Installing mlpack +## Installing mlpack Installing the mlpack bindings for Python is straightforward. It's easy to use -conda or pip to do this: +`conda` or `pip` to do this: -@code{.sh} +```sh pip install mlpack -@endcode +``` -@code{.sh} +```sh conda install -c conda-forge mlpack -@endcode - -Otherwise, we can build the Python bindings from scratch, as follows. First we -have to install the dependencies (the code below is for Ubuntu), then we can -build and install mlpack. You can copy-paste the commands into your shell. - -@code{.sh} -sudo apt-get install g++ cmake libarmadillo-dev python-pip wget -sudo pip install cython setuptools distutils numpy pandas -wget https://www.mlpack.org/files/mlpack-3.4.2.tar.gz -tar -xvzpf mlpack-3.4.2.tar.gz -mkdir -p mlpack-3.4.2/build/ && cd mlpack-3.4.2/build/ -cmake ../ && make -j4 && sudo make install -@endcode - -More information on the build process and details can be found on the @ref build -page. You may also need to set the environment variable @c LD_LIBRARY_PATH to -include @c /usr/local/lib/ on most Linux systems. - -@code -export LD_LIBRARY_PATH=/usr/local/lib/ -@endcode +``` You can also use the mlpack Docker image on Dockerhub, which has all of the Python bindings pre-installed: -@code +```sh docker run -it mlpack/mlpack /bin/bash -@endcode +``` -@section python_quickstart_example Simple mlpack quickstart example +Otherwise, you can build the Python bindings from scratch using the +documentation in the [main README](../../README.md). + +## Simple mlpack quickstart example As a really simple example of how to use mlpack from Python, let's do some -simple classification on a subset of the standard machine learning @c covertype +simple classification on a subset of the standard machine learning `covertype` dataset. We'll first split the dataset into a training set and a testing set, then we'll train an mlpack random forest on the training data, and finally we'll print the accuracy of the random forest on the test dataset. You can copy-paste this code directly into Python to run it. -@code{.py} +```py import mlpack import pandas as pd import numpy as np @@ -104,38 +80,26 @@ correct = np.sum( output['predictions'] == np.reshape(test_labels, (test_labels.shape[0],))) print(str(correct) + ' correct out of ' + str(len(test_labels)) + ' (' + str(100 * float(correct) / float(len(test_labels))) + '%).') -@endcode +``` We can see that we achieve reasonably good accuracy on the test dataset (80%+); -if we use the full @c covertype.csv.gz, the accuracy should increase +if we use the full `covertype.csv.gz`, the accuracy should increase significantly (but training will take longer). It's easy to modify the code above to do more complex things, or to use different mlpack learners, or to interface with other machine learning toolkits. -@section python_quickstart_whatelse What else does mlpack implement? - -The example above has only shown a little bit of the functionality of mlpack. -Lots of other commands are available with different functionality. A full list -of each of these commands and full documentation can be found on the following -page: - - - Python documentation - -For more information on what mlpack does, see https://www.mlpack.org/. -Next, let's go through another example for providing movie recommendations with -mlpack. - -@section python_quickstart_movierecs Using mlpack for movie recommendations +## Using mlpack for movie recommendations In this example, we'll train a collaborative filtering model using mlpack's -cf() method. We'll train this on the MovieLens dataset from -https://grouplens.org/datasets/movielens/, and then we'll use the model that we -train to give recommendations. +[`cf()`](https://www.mlpack.org/doc/stable/python_documentation.html#cf) method. +We'll train this on the +[MovieLens dataset](https://grouplens.org/datasets/movielens/), and then we'll +use the model that we train to give recommendations. You can copy-paste this code directly into Python to run it. -@code{.py} +```py import mlpack import pandas as pd import numpy as np @@ -170,12 +134,12 @@ print("Recommendations for user 1:") for i in range(10): print(" " + str(i) + ": " + str(movies.loc[movies['movieId'] == output['output'][0, i]].iloc[0]['title'])) -@endcode +``` Here is some example output, showing that user 1 seems to have good taste in movies: -@code{.unparsed} +``` Recommendations for user 1: 0: Casablanca (1942) 1: Pan's Labyrinth (Laberinto del fauno, El) (2006) @@ -187,29 +151,19 @@ Recommendations for user 1: 7: Out for Justice (1991) 8: Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1964) 9: Schindler's List (1993) -@endcode +``` -@section python_quickstart_nextsteps Next steps with mlpack +## Next steps with mlpack Now that you have done some simple work with mlpack, you have seen how it can -easily plug into a data science workflow in Python. A great thing to do next -would be to look at more documentation for the Python mlpack bindings: +easily plug into a data science workflow in Python. But the two examples above +have only shown a little bit of the functionality of mlpack. Lots of other +commands are available with different functionality. A full list of each of +these commands and full documentation can be found on the following page: - - Python mlpack - binding documentation + - [Python documentation](https://www.mlpack.org/doc/stable/python_documentation.html) Also, mlpack is much more flexible from C++ and allows much greater functionality. So, more complicated tasks are possible if you are willing to write C++ (or perhaps Cython). To get started learning about mlpack in C++, the -following resources might be helpful: - - - mlpack - C++ tutorials - - mlpack - build and installation guide - - Simple - sample C++ mlpack programs - - mlpack - Doxygen documentation homepage - - */ +[C++ quickstart](cpp.md) would be a good place to go. diff --git a/doc/tutorials/README.md b/doc/tutorials/README.md index fd60ffd11d..b91bd0c3fe 100644 --- a/doc/tutorials/README.md +++ b/doc/tutorials/README.md @@ -1,40 +1,18 @@ - ## Tutorials -Tutorials for mlpack can be found [here : mlpack tutorials](https://www.mlpack.org/doc/mlpack-git/doxygen/tutorials.html). +Tutorials for mlpack can be found in this directory. - -### General mlpack tutorials - -These tutorials introduce the basic concepts of working with mlpack, aimed at developers who want to use and contribute to mlpack but are not sure where to start. - -* [Building mlpack from source](https://www.mlpack.org/doc/mlpack-git/doxygen/build.html) -* [File Formats in mlpack](https://www.mlpack.org/doc/mlpack-git/doxygen/formatdoc.html) -* [Matrices in mlpack](https://www.mlpack.org/doc/mlpack-git/doxygen/matrices.html) -* [mlpack input and output](https://www.mlpack.org/doc/mlpack-git/doxygen/iodoc.html) -* [mlpack timers](https://www.mlpack.org/doc/mlpack-git/doxygen/timer.html) -* [Simple sample mlpack programs](https://www.mlpack.org/doc/mlpack-git/doxygen/sample.html) - - -### Method-specific tutorials - -These tutorials introduce the various methods mlpack offers, aimed at users who want to get started quickly. These tutorials start with simple examples and progress to complex, extensible uses. - -* [NeighborSearch tutorial (mlpack_knn / mlpack_kfn)](https://www.mlpack.org/doc/mlpack-git/doxygen/nstutorial.html) -* [LinearRegression tutorial (mlpack_linear_regression)](https://www.mlpack.org/doc/mlpack-git/doxygen/lrtutorial.html) -* [RangeSearch tutorial (mlpack_range_search)](https://www.mlpack.org/doc/mlpack-git/doxygen/rstutorial.html) -* [Density Estimation Trees tutorial (mlpack_det)](https://www.mlpack.org/doc/mlpack-git/doxygen/dettutorial.html) -* [K-Means tutorial (mlpack_kmeans)](https://www.mlpack.org/doc/mlpack-git/doxygen/kmtutorial.html) -* [FastMKS tutorial (mlpack_fastmks)](https://www.mlpack.org/doc/mlpack-git/doxygen/fmkstutorial.html) -* [Euclidean Minimum Spanning Trees tutorial (mlpack_emst)](https://www.mlpack.org/doc/mlpack-git/doxygen/emst_tutorial.html) -* [Alternating Matrix Factorization Tutorial](https://www.mlpack.org/doc/mlpack-git/doxygen/amftutorial.html) -* [Collaborative Filtering Tutorial](https://www.mlpack.org/doc/mlpack-git/doxygen/cftutorial.html) - - -### Policy Class Documentation - -mlpack uses templates to achieve its genericity and flexibility. Some of the template types used by mlpack are common across multiple machine learning algorithms. The links below provide documentation for some of these common types. - -* [The MetricType policy in mlpack](https://www.mlpack.org/doc/mlpack-git/doxygen/metrics.html) -* [The KernelType policy in mlpack](https://www.mlpack.org/doc/mlpack-git/doxygen/kernels.html) -* [The TreeType policy in mlpack](https://www.mlpack.org/doc/mlpack-git/doxygen/trees.html) + - [Alternating Matrix Factorization (AMF)](amf.md) + - [Artificial Neural Networks (ANN)](ann.md) + - [Approximate k-Furthest Neighbor Search (`approx_kfn`)](approx_kfn.md) + - [Collaborative Filtering (CF)](cf.md) + - [DatasetMapper](datasetmapper.md) + - [Density Estimation Trees (DET)](det.md) + - [Euclidean Minimum Spanning Trees (EMST)](emst.md) + - [Fast Max-Kernel Search (FastMKS)](fastmks.md) + - [Image Utilities](image.md) + - [k-Means Clustering](kmeans.md) + - [Linear Regression](linear_regression.md) + - [Neighbor Search (k-Nearest-Neighbors)](neighbor_search.md) + - [Range Search](range_search.md) + - [Reinforcement Learning](reinforcement_learning.md) diff --git a/doc/tutorials/amf.md b/doc/tutorials/amf.md new file mode 100644 index 0000000000..c9add125d5 --- /dev/null +++ b/doc/tutorials/amf.md @@ -0,0 +1,183 @@ +# Alternating Matrix Factorization tutorial + +Alternating matrix factorization decomposes a matrix `V` in the form `V ~ WH` +where `W` is called the basis matrix and `H` is called the encoding matrix.. `V` +is taken to be of size `n x m` and the obtained `W` is `n x r` and `H` is `r x +m`. The size `r` is called the *rank* of the factorization. Factorization is +done by alternately calculating `W` and `H` respectively while holding the other +matrix constant. + +mlpack provides a simple C++ interface to perform Alternating Matrix +Factorization. + +## The `AMF` class + +The `AMF` class is templatized with 3 parameters; the first contains the policy +used to determine when the algorithm has converged; the second contains the +initialization rule for the `W` and `H` matrix; the last contains the update +rule to be used during each iteration. This templatization allows the user to +try various update rules, initialization rules, and termination policies +(including ones not supplied with mlpack) for factorization. + +The class provides the following method that performs factorization + +```c++ +template double Apply(const MatType& V, + const size_t r, + arma::mat& W, + arma::mat& H); +``` + +## Using different termination policies + +The `AMF` implementation comes with different termination policies to support +many implemented algorithms. Every termination policy implements the following +method which returns the status of convergence. + +```c++ +bool IsConverged(arma::mat& W, arma::mat& H) +``` + +Below is a list of all the termination policies that mlpack contains. + + - `mlpack::amf::SimpleResidueTermination` + - `mlpack::amf::SimpleToleranceTermination` + - `mlpack::amf::ValidationRMSETermination` + +In `SimpleResidueTermination`, the termination decision depends on two factors, +value of residue and number of iteration. If the current value of residue drops +below the threshold or the number of iterations goes beyond the threshold, +positive termination signal is passed to AMF. + +In `SimpleToleranceTermination`, termination criterion is met when the increase +in residue value drops below the given tolerance. To accommodate spikes, certain +number of successive residue drops are accepted. Secondary termination criterion +terminates algorithm when iteration count goes beyond the threshold. + +`ValidationRMSETermination` divides the data into 2 sets, training set and +validation set. Entries of the validation set are nullifed in the input matrix. +Termination criterion is met when increase in validation set RMSe value drops +below the given tolerance. To accommodate spikes certain number of successive +validation RMSE drops are accepted. This upper imit on successive drops can be +adjusted with `reverseStepCount`. A secondary termination criterion terminates +the algorithm when the iteration count goes above the threshold. Though this +termination policy is better measure of convergence than the above 2 termination +policies, it may cause a decrease in performance since it is computationally +expensive. + +On the other hand, `CompleteIncrementalTermination` and +`IncompleteIncrementalTermination` are just wrapper classes for other +termination policies. These policies are used when AMF is applied with +`SVDCompleteIncrementalLearning` and `SVDIncompleteIncrementalLearning`, +respectively. + +## Using different initialization policies + +mlpack currently has 2 initialization policies implemented for AMF: + + - `mlpack::amf::RandomInitialization` + - `mlpack::amf::RandomAcolInitialization` + +`RandomInitialization` initializes matrices `W` and `H` with random uniform +distribution while `RandomAcolInitialization` initializes the `W` matrix by +averaging p randomly chosen columns of `V`. In the case of +`RandomAcolInitialization`, `p` is a template parameter. + +To implement their own initialization policy, users need to define the following +function in their class. + +```c++ +template +inline static void Initialize(const MatType& V, + const size_t r, + arma::mat& W, + arma::mat& H) +``` + +## Using different update rules + +mlpack implements the following update rules for the AMF class: + + - `mlpack::amf::NMFALSUpdate` + - `mlpack::amf::NMFMultiplicativeDistanceUpdate` + - `mlpack::amf::NMFMultiplicativeDivergenceUpdate` + - `mlpack::amf::SVDBatchLearning` + - `mlpack::amf::SVDIncompleteIncrementalLearning` + - `mlpack::amf::SVDCompleteIncrementalLearning` + +Non-Negative Matrix factorization can be achieved with `NMFALSUpdate`, +`NMFMultiplicativeDivergenceUpdate` or `NMFMultiplicativeDivergenceUpdate`. +`NMFALSUpdate` implements a simple Alternating Least Squares optimization while +the other rules implement algorithms given in the paper 'Algorithms for +Non-negative Matrix Factorization'. + +The remaining update rules perform the singular value decomposition of the +matrix `V`. This SVD factorization is optimized for use by mlpack's +collaborative filtering code (see the [collaborative filtering +tutorial](cf.md)). This use of SVD factorizers for collaborative filtering is +described in the paper 'A Guide to Singular Value Decomposition for +Collaborative Filtering' by Chih-Chao Ma. For further details about the +algorithms refer to the respective class documentation. + +## Using Non-Negative Matrix Factorization with `AMF` + +The use of `AMF` for Non-Negative Matrix factorization is simple. The AMF module +defines `mlpack::amf::NMFALSFactorizer` which can be used directly without +knowing the internal structure of `AMF`. For example: + +```c++ +#include + +using namespace std; +using namespace arma; +using namespace mlpack::amf; + +int main() +{ + NMFALSFactorizer nmf; + mat W, H; + mat V = randu(100, 100); + double residue = nmf.Apply(V, W, H); +} +``` + +`NMFALSFactorizer` uses `SimpleResidueTermination`, which is most preferred with +Non-Negative Matrix factorizers. The initialization of `W` and `H` in +`NMFALSFactorizer` is random. The `Apply()` function returns the residue +obtained by comparing the constructed matrix `W * H` with the original matrix +`V`. + +## Using Singular Value Decomposition with `AMF` + +mlpack has the following SVD factorizers implemented for AMF: + + - `mlpack::amf::SVDBatchFactorizer` + - `mlpack::amf::SVDIncompleteIncrementalFactorizer` + - `mlpack::amf::SVDCompleteIncrementalFactorizer` + +Each of these factorizers takes a template parameter `MatType`, which specifies +the type of the matrix `V` (dense or sparse---these have types `arma::mat` and +`arma::sp_mat`, respectively). When the matrix to be factorized is relatively +sparse, specifying `MatType = arma::sp_mat` can provide a runtime boost. + +```c++ +#include + +using namespace std; +using namespace arma; +using namespace mlpack::amf; + +int main() +{ + sp_mat V = randu(100,100); + mat W, H; + + SVDBatchFactorizer svd; + double residue = svd.Apply(V, W, H); +} +``` + +## Further documentation + +For further documentation on the `AMF` class, consult the `mlpack::amf::AMF` +source code comments. diff --git a/doc/tutorials/amf/amf.txt b/doc/tutorials/amf/amf.txt deleted file mode 100644 index 373c2b804c..0000000000 --- a/doc/tutorials/amf/amf.txt +++ /dev/null @@ -1,210 +0,0 @@ -/*! - -@file amf.txt -@author Sumedh Ghaisas -@brief Tutorial for how to use the AMF class - -@page amftutorial Alternating Matrix Factorization tutorial - -@section intro_amftut Introduction - -Alternating Matrix Factorization - -Alternating matrix factorization decomposes matrx V in the form \f$ V \approx WH \f$ -where W is called the basis matrix and H is called the encoding matrix. V is -taken to be of size n x m and the obtained W is n x r and H is r x m. The size -r is called the rank of the factorization. Factorization is done by alternately -calculating W and H respectively while holding the other matrix constant. - -\b mlpack provides: - - - a \ref amf_amftut "simple C++ interface" to perform Alternating Matrix Factorization - -@section toc_amftut Table of Contents - -A list of all the sections this tutorial contains. - - - \ref intro_amftut - - \ref toc_amftut - - \ref amf_amftut - - \ref t_policy_amftut - - \ref init_rule_amftut - - \ref update_rule_amftut - - \ref nmf_amftut - - \ref svd_amftut - - \ref further_doc_amftut - -@section amf_amftut The 'AMF' class - -The AMF class is templatized with 3 parameters; the first contains the policy -used to determine when the algorithm has converged; the second contains the -initialization rule for the W and H matrix; the last contains the update rule -to be used during each iteration. This templatization allows the user to try -various update rules, initialization rules, and termination policies (including -ones not supplied with mlpack) for factorization. - -The class provides the following method that performs factorization -@code -template double Apply(const MatType& V, - const size_t r, - arma::mat& W, - arma::mat& H); -@endcode - -@subsection t_policy_amftut Using different termination policies - -The AMF implementation comes with different termination policies to support many -implemented algorithms. Every termination policy implements the following method -which returns the status of convergence. -@code -bool IsConverged(arma::mat& W, arma::mat& H) -@endcode - -Below is a list of all the termination policies that mlpack contains. - - - \ref mlpack::amf::SimpleResidueTermination - - \ref mlpack::amf::SimpleToleranceTermination - - \ref mlpack::amf::ValidationRMSETermination - -In \c SimpleResidueTermination, termination decision depends on two factors, value -of residue and number of iteration. If the current value of residue drops below -the threshold or the number of iterations goes beyond the threshold, positive -termination signal is passed to AMF. - -In \c SimpleToleranceTermination, termination criterion is met when the increase -in residue value drops below the given tolerance. To accommodate spikes, certain -number of successive residue drops are accepted. Secondary termination criterion -terminates algorithm when iteration count goes beyond the threshold. - -\c ValidationRMSETermination divides the data into 2 sets, training set and -validation set. Entries of validation set are nullifed in the input matrix. -Termination criterion is met when increase in validation set RMSe value drops -below the given tolerance. To accommodate spikes certain number of successive -validation RMSE drops are accepted. This upper imit on successive drops can be -adjusted with \c reverseStepCount. A secondary termination criterion terminates -the algorithm when the iteration count goes above the threshold. Though this -termination policy is better measure of convergence than the above 2 termination -policies, it may cause a decrease in performance since it is computationally -expensive. - -On the other hand, \ref mlpack::amf::CompleteIncrementalTermination -"CompleteIncrementalTermination" and \ref mlpack::amf::IncompleteIncrementalTermination -"IncompleteIncrementalTermination" are just wrapper classes for other -termination policies. These policies are used when AMF is applied with -\ref mlpack::amf::SVDCompleteIncrementalLearning -"SVDCompleteIncrementalLearning" and -\ref mlpack::amf::SVDIncompleteIncrementalLearning -"SVDIncompleteIncrementalLearning", respectively. - -@subsection init_rule_amftut Using different initialization policies - -mlpack currently has 2 initialization policies implemented for AMF: - - - \ref mlpack::amf::RandomInitialization "RandomInitialization" - - \ref mlpack::amf::RandomAcolInitialization "RandomAcolInitialization" - -\c RandomInitialization initializes matrices W and H with random uniform -distribution while \c RandomAcolInitialization initializes the W matrix by -averaging p randomly chosen columns of V. In the case of -\c RandomAcolInitialization, p is a template parameter. - -To implement their own initialization policy, users need to define the following -function in their class. - -@code -template -inline static void Initialize(const MatType& V, - const size_t r, - arma::mat& W, - arma::mat& H) -@endcode - -@subsection update_rule_amftut Using different update rules - -mlpack implements the following update rules for the AMF class: - - - \ref mlpack::amf::NMFALSUpdate "AMFALSUpdate" - - \ref mlpack::amf::NMFMultiplicativeDistanceUpdate "NMFMultiplicativeDistanceUpdate" - - \ref mlpack::amf::NMFMultiplicativeDivergenceUpdate "NMFMultiplicativeDivergenceUpdate" - - \ref mlpack::amf::SVDBatchLearning "SVDBatchLearning" - - \ref mlpack::amf::SVDIncompleteIncrementalLearning "SVDIncompleteIncrementalLearning" - - \ref mlpack::amf::SVDCompleteIncrementalLearning "SVDCompleteIncrementalLearning" - -Non-Negative Matrix factorization can be achieved with \c NMFALSUpdate, -\c NMFMultiplicativeDivergenceUpdate or \c NMFMultiplicativeDivergenceUpdate. -\c NMFALSUpdate implements a simple Alternating Least Squares optimization while -the other rules implement algorithms given in the paper 'Algorithms for -Non-negative Matrix Factorization'. - -The remaining update rules perform the singular value decomposition of the matrix V. -This SVD factorization is optimized for use by mlpack's collaborative filtering -code (\ref cftutorial). This use of SVD factorizers for collaborative filtering -is described in the paper 'A Guide to Singular Value Decomposition for -Collaborative Filtering' by Chih-Chao Ma. For further details about the -algorithms refer to the respective class documentation. - -@subsection nmf_amftut Using Non-Negative Matrix Factorization with AMF - -The use of AMF for Non-Negative Matrix factorization is simple. The AMF module -defines \ref mlpack::amf::NMFALSFactorizer "NMFALSFactorizer" which can be used -directly without knowing the internal structure of AMF. For example: - -@code -#include -#include - -using namespace std; -using namespace arma; -using namespace mlpack::amf; - -int main() -{ - NMFALSFactorizer nmf; - mat W, H; - mat V = randu(100, 100); - double residue = nmf.Apply(V, W, H); -} -@endcode - -\c NMFALSFactorizer uses \c SimpleResidueTermination, which is most preferred -with Non-Negative Matrix factorizers. The initialization of W and H in -\c NMFALSFactorizer is random. The \c Apply() function returns the residue -obtained by comparing the constructed matrix W * H with the original matrix V. - -@subsection svd_amftut Using Singular Value Decomposition with AMF - -mlpack has the following SVD factorizers implemented for AMF: - - - \ref mlpack::amf::SVDBatchFactorizer "SVDBatchFactorizer" - - \ref mlpack::amf::SVDIncompleteIncrementalFactorizer "SVDIncompleteIncrementalFactorizer" - - \ref mlpack::amf::SVDCompleteIncrementalFactorizer "SVDCompleteIncrementalFactorizer" - -Each of these factorizers takes a template parameter \c MatType, which specifies -the type of the matrix V (dense or sparse---these have types \c arma::mat and -\c arma::sp_mat, respectively). When the matrix to be factorized is relatively -sparse, specifying \c MatType \c = \c arma::sp_mat can provide a runtime boost. - -@code -#include -#include - -using namespace std; -using namespace arma; -using namespace mlpack::amf; - -int main() -{ - sp_mat V = randu(100,100); - mat W, H; - - SVDBatchFactorizer svd; - double residue = svd.Apply(V, W, H); -} -@endcode - -@section further_doc_amftut Further documentation - -For further documentation on the AMF class, consult the \ref mlpack::amf::AMF -"complete API documentation". - -*/ diff --git a/doc/tutorials/ann/ann.txt b/doc/tutorials/ann.md similarity index 59% rename from doc/tutorials/ann/ann.txt rename to doc/tutorials/ann.md index e3047f0be4..8bda0a5043 100644 --- a/doc/tutorials/ann/ann.txt +++ b/doc/tutorials/ann.md @@ -1,11 +1,4 @@ -/*! -@file ann.txt -@author Marcus Edel (https://kurg.org) -@brief Tutorial for how to use the neural network code in mlpack. - -@page anntutorial Neural Network tutorial - -@section intro_anntut Introduction +# Neural Network tutorial There is vast literature on neural networks and their uses, as well as strategies for choosing initial points effectively, keeping the algorithm from @@ -14,34 +7,21 @@ optimizers, and so forth. mlpack implements many of these building blocks, making it very easy to create different neural networks in a modular way. mlpack currently implements two easy-to-use forms of neural networks: -\b Feed-Forward \b Networks (this includes convolutional neural networks) and -\b Recurrent \b Neural \b Networks. +*Feed-Forward Networks* (this includes convolutional neural networks) and +*Recurrent Neural Networks*. -@section toc_anntut Table of Contents - -This tutorial is split into the following sections: - - - \ref intro_anntut - - \ref toc_anntut - - \ref model_api_anntut - - \ref layer_api_anntut - - \ref model_setup_training_anntut - - \ref model_saving_loading_anntut - - \ref extracting_parameters_anntut - - \ref further_anntut - -@section model_api_anntut Model API +## Model API There are two main neural network classes that are meant to be used as container for neural network layers that \b mlpack implements; each class is suited to a different setting: -- \c FFN: the Feed Forward Network model provides a means to plug layers +- `FFN`: the Feed Forward Network model provides a means to plug layers together in a feed-forward fully connected manner. This is the 'standard' type of deep learning model, and includes convolutional neural networks (CNNs). -- \c RNN: the Recurrent Neural Network model provides a means to consider +- `RNN`: the Recurrent Neural Network model provides a means to consider successive calls to forward as different time-steps in a sequence. This is often used for time sequence modeling tasks, such as predicting the next character in a sequence. @@ -51,7 +31,7 @@ Below is some basic guidance on what should be used. Note that the question of guidance below is just that---guidance---and may not be right for a particular problem. - - \b Feed-forward \b Networks allow signals or inputs to travel one way only. + - *Feed-forward Networks* allow signals or inputs to travel one way only. There is no feedback within the network; for instance, the output of any layer does only affect the upcoming layer. That makes Feed-Forward Networks straightforward and very effective. They are extensively used in pattern @@ -59,72 +39,72 @@ problem. set of input and one or more output variables. - - \b Recurrent \b Networks allow signals or inputs to travel in both directions by - introducing loops in the network. Computations derived from earlier inputs are - fed back into the network, which gives the recurrent network some kind of + - *Recurrent Networks* allow signals or inputs to travel in both directions by + introducing loops in the network. Computations derived from earlier inputs + are fed back into the network, which gives the recurrent network some kind of memory. RNNs are currently being used for all kinds of sequential tasks; for instance, time series prediction, sequence labeling, and sequence classification. -In order to facilitate consistent implementations, the \c FFN and \c RNN classes +In order to facilitate consistent implementations, the `FFN` and `RNN` classes have a number of methods in common: - - \c Train(): trains the initialized model on the given input data. Optionally + - `Train()`: trains the initialized model on the given input data. Optionally an optimizer object can be passed to control the optimization process. - - \c Predict(): predicts the responses to a given set of predictors. Note the + - `Predict()`: predicts the responses to a given set of predictors. Note the responses will reflect the output of the specified output layer. - - \c Add(): this method can be used to add a layer to the model. + - `Add()`: this method can be used to add a layer to the model. -@note -To be able to optimize the network, both classes implement the OptimizerFunction -API. In short, the \c FNN and \c RNN class implement two methods: \c Evaluate() -and \c Gradient(). This enables the optimization given some learner and some -performance measure. +*Note*: to be able to optimize the network, both classes implement the +[ensmallen](https://www.ensmallen.org) function API. In short, the `FNN` and +`RNN` class implement two methods: `Evaluate()` and `Gradient()`. This enables +the optimization given some learner and some performance measure. -Similar to the existing layer infrastructure, the \c FFN and \c RNN classes are +Similar to the existing layer infrastructure, the `FFN` and `RNN` classes are very extensible, having the following template arguments; which can be modified to change the behavior of the network: - - \c OutputLayerType: this type defines the output layer used to evaluate the - network; by default, \c NegativeLogLikelihood is used. + - `OutputLayerType`: this type defines the output layer used to evaluate the + network; by default, `NegativeLogLikelihood` is used. - - \c InitializationRuleType: this type defines the method by which initial - parameters are set; by default, \c RandomInitialization is used. + - `InitializationRuleType`: this type defines the method by which initial + parameters are set; by default, `RandomInitialization` is used. -@code +```c++ template< typename OutputLayerType = NegativeLogLikelihood<>, typename InitializationRuleType = RandomInitialization > class FNN; -@endcode +``` -Internally, the \c FFN and \c RNN class keeps an instantiated \c OutputLayerType +Internally, the `FFN` and `RNN` class keeps an instantiated `OutputLayerType` class (which can be given in the constructor). This is useful for using different loss functions like the Negative-Log-Likelihood function or the \c VRClassReward function, which takes an optional score parameter. Therefore, you -can write a non-static OutputLayerType class and use it seamlessly in -combination with the \c FNN and \c RNN class. The same applies to the \c -InitializationRuleType template parameter. +can write a non-static `OutputLayerType` class and use it seamlessly in +combination with the `FNN` and `RNN` class. The same applies to the +`InitializationRuleType` template parameter. By choosing different components for each of these template classes in -conjunction with the \c Add() method, a very arbitrary network object can be +conjunction with the `Add()` method, a very arbitrary network object can be constructed. -Below are several examples of how the \c FNN and \c RNN classes might be used. -The first examples focus on the \c FNN class, and the last shows how the \c -RNN class can be used. +Below are several examples of how the `FNN` and `RNN` classes might be used. +The first examples focus on the `FNN` class, and the last shows how the +`RNN` class can be used. -The simplest way to use the FNN<> class is to pass in a dataset with the +The simplest way to use the `FNN` class is to pass in a dataset with the corresponding labels, and receive the classification in return. Note that the dataset must be column-major – that is, one column corresponds to one point. See -the \ref matrices "matrices guide" for more information. +the [matrices guide](../user/matrices.md) for more information. The code below builds a simple feed-forward network with the default options, -then queries for the assignments for every point in the \c queries matrix. +then queries for the assignments for every point in the `queries` matrix. +TODO: move to separate rendered image \dot digraph G { fontname = "Hilda 10" @@ -188,15 +168,13 @@ digraph G { l36 -> l42 l37 -> l40 l37 -> l41 l37 -> l42 } \enddot -@note -The number of inputs in the above graph doesn't match with the real + +*Note*: the number of inputs in the above graph doesn't match with the real number of features in the thyroid dataset and are just used as an abstract representation. -@code -#include -#include -#include +```c++ +#include using namespace mlpack; using namespace mlpack::ann; @@ -262,16 +240,16 @@ int main() std::cout << "Classification Error for the Test set: " << classificationError << std::endl; return 0; } -@endcode +``` Now, the matrix prediction holds the classification of each point in the dataset. Subsequently, we find the classification error by comparing it -with testLabels. +with `testLabels`. In the next example, we create simple noisy sine sequences, which are trained -later on, using the RNN class in the `RNNModel()` method. +later on, using the `RNN` class in the `RNNModel()` method. -@code +```c++ void GenerateNoisySines(arma::mat& data, arma::mat& labels, const size_t points, @@ -346,240 +324,243 @@ void RNNModel() StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); model.Train(input, labels, opt); } -@endcode +``` For further examples on the usage of the ann classes, see [mlpack models](https://github.com/mlpack/models). -@section layer_api_anntut Layer API +## Layer API -In order to facilitate consistent implementations, we have defined a LayerType -API that describes all the methods that a \c layer may implement. mlpack offers -a few variations of this API, each designed to cover some of the model -characteristics mentioned in the previous section. Any \c layer requires the -implementation of a \c Forward() method. The interface looks like: +In order to facilitate consistent implementations, we have defined a `LayerType` +API that describes all the methods that a layer may implement. mlpack offers a +few variations of this API, each designed to cover some of the model +characteristics mentioned in the previous section. Any layer requires the +implementation of a `Forward()` method. The interface looks like: -@code -template -void Forward(const arma::Mat& input, arma::Mat& output); -@endcode +```c++ +template +void Forward(const MatType& input, MatType& output); +``` + +*(Note that `MatType` can be a template parameter of the layer class itself, not +necessarily the `Forward()` function. This applies to the other functions of +the API too.)* The method should calculate the output of the layer given the input matrix and -store the result in the given output matrix. Next, any \c layer must implement -the Backward() method, which uses certain computations obtained during the -forward pass and should calculate the function f(x) by propagating x backward -through f: +store the result in the given output matrix. Next, any layer must implement the +`Backward()` method, which uses certain computations obtained during the forward +pass and should calculate the function `f(x)` by propagating `x` backward +through `f`: -@code -template -void Backward(const arma::Mat& input, - const arma::Mat& gy, - arma::Mat& g); -@endcode +```c++ +template +void Backward(const MatType& input, + const MatType& gy, + MatType& g); +``` Finally, if the layer is differentiable, the layer must also implement -a Gradient() method: +a `Gradient()` method: -@code -template -void Gradient(const arma::Mat& input, - const arma::Mat& error, - arma::Mat& gradient); -@endcode +```c++ +template +void Gradient(const MatType& input, + const MatType& error, + MatType& gradient); +``` -The Gradient function should calculate the gradient with respect to the input -activations \c input and calculated errors \c error and place the results into -the gradient matrix object \c gradient that is passed as an argument. +The `Gradient()` function should calculate the gradient with respect to the +input activations `input` and calculated errors `error` and place the results +into the gradient matrix object `gradient` that is passed as an argument. -@note -Note that each method accepts a template parameter InputType, OutputType -or GradientType, which may be arma::mat (dense Armadillo matrix) or arma::sp_mat +Each of these three methods accepts a template parameter `MatType`, +which may be `arma::mat` (dense Armadillo matrix) or `arma::sp_mat` (sparse Armadillo matrix). This allows support for both sparse-supporting and -non-sparse-supporting \c layer without explicitly passing the type. +non-sparse-supporting layer without explicitly passing the type. -In addition, each layer must implement the Parameters(), InputParameter(), -OutputParameter(), Delta() methods, differentiable layer should also provide -access to the gradient by implementing the Gradient(), Parameters() member -function. Note each function is a single line that looks like: +Every new layer should inherit from `mlpack::ann::Layer`, which defines +some core functionality. There are three additional functions that must be +implemented: -@code -OutputDataType const& Parameters() const { return weights; } -@endcode + - `void ComputeOutputDimensions()`: this sets the internal member + `outputDimensions` to the correct output dimensions of the layer, given that + `inputDimensions` is set. + + - `size_t WeightSize() const`: given that `ComputeOutputDimensions()` has been + called (and so `outputDimensions` and `inputDimensions` are correct), return + the number of trainable weights in the layer. + + - `void SetWeights(typename MatType::elem_type*)`: this sets the layer's + internal parameter memory to the given pointer. Below is an example that shows each function with some additional boilerplate -code. - -@note -Note this is not an actual layer but instead an example that exists to show and +code. Note this is not an actual layer but instead an example that exists to show and document all the functions that mlpack layer must implement. For a better -overview of the various layers, see \ref mlpack::ann. Also be aware that the -implementations of each of the methods in this example are entirely fake and do -not work; this example exists for its API, not its implementation. +overview of the various layers, see the layers in +`src/mlpack/methods/ann/layer/`. Also be aware that the implementations of each +of the methods in this example are entirely fake and do not work; this example +exists for its API, not its implementation. -Note that layer sometimes have different properties. These properties are -known at compile-time through the mlpack::ann::LayerTraits class, and some -properties may imply the existence (or non-existence) of certain functions. -Refer to the LayerTraits @ref layer_traits.hpp for more documentation on that. - -The two template parameters below must be template parameters to the layer, in -the order given below. More template parameters are fine, but they must come -after the first two. - - - \c InputDataType: this defines the internally used input type for example to - store the parameter matrix. Note, a layer could be built on a dense matrix or - a sparse matrix. All mlpack trees should be able to support any Armadillo- - compatible matrix type. When the layer is written it should be assumed that - MatType has the same functionality as arma::mat. Note that - - - \c OutputDataType: this defines the internally used input type for example to - store the parameter matrix. Note, a layer could be built on a dense matrix or - a sparse matrix. All mlpack trees should be able to support any Armadillo- - compatible matrix type. When the layer is written it should be assumed that - MatType has the same functionality as arma::mat. - -@code -template -class ExampleLayer +```c++ +template +class ExampleLayer : public Layer { public: - ExampleLayer(const size_t inSize, const size_t outSize) : - inputSize(inSize), outputSize(outSize) + // Note that the input size will be set in the member + // `Layer::inputParameters` automatically before + // `ComputeOutputDimensions()` is called. + ExampleLayer() { /* Nothing to do here */ } + + private: + MatType weights; } -@endcode +``` -The constructor for \c ExampleLayer will build the layer given the input and -output size. Note that, if the input or output size information isn't used -internally it's not necessary to provide a specific constructor. Also, one could -add additional or other information that are necessary for the layer -construction. One example could be: +The constructor for `ExampleLayer` will build the layer given the output size. +Note that, if the output size information isn't used internally it's not +necessary to provide a specific constructor. Also, one could add additional or +other information that are necessary for the layer construction. One example +could be: -@code -ExampleLayer(const double ratio = 0.5) : ratio(ratio) {/* Nothing to do here*/} -@endcode +```c++ +template +ExampleLayer(const double ratio = 0.5) : ratio(ratio) +{ /* Nothing to do here */ } +``` -When this constructor is finished, the entire layer will be built and is ready -to be used. Next, as pointed out above, each layer has to follow the LayerType -API, so we must implement some additional functions. +When this constructor is finished, the entire layer will be built, but may not +yet be ready to use. We can assume that the enclosing `FFN` or `RNN` network +will call `ComputeOutputDimensions()`, `WeightSize()`, and `SetWeights()` before +any call to `Forward()` is done. Next, as pointed out above, each layer has to +follow the `LayerType` API, so we must implement some additional functions. -@code -template -void Forward(const InputType& input, OutputType& output) +```c++ +template +void ExampleLayer::Forward(const MatType& input, MatType& output) { - output = arma::ones(input.n_rows, input.n_cols); + output = arma::ones(input.n_rows, input.n_cols) + weights; } -template -void Backward(const InputType& input, const ErrorType& gy, GradientType& g) +template +void ExampleLayer::Backward(const MatType& input, + const MatType& gy, + MatType& g) { - g = arma::zeros(gy.n_rows, gy.n_cols) + gy; + g = gy - weights; } -template -void Gradient(const InputType& input, - ErrorType& error, - GradientType& gradient) +template +void ExampleLayer::Gradient(const InputType& input, + ErrorType& error, + GradientType& gradient) { - gradient = arma::zeros(input.n_rows, input.n_cols) * error; + gradient = arma::ones(input.n_rows, input.n_cols); } -@endcode +``` -The three functions \c Forward(), \c Backward() and \c Gradient() (which is -needed for a differentiable layer) contain the main logic of the layer. The -following functions are just to access and manipulate the different layer -parameters. +The three functions `Forward()`, `Backward()` and `Gradient()` (which is +needed for a differentiable layer) contain the main logic of the layer. -@code -OutputDataType& Parameters() { return weights; } -InputDataType& InputParameter() { return inputParameter; } -OutputDataType& OutputParameter() { return outputParameter; } -OutputDataType& Delta() { return delta; } -OutputDataType& Gradient() { return gradient; } -@endcode +Now let's implement `ComputeOutputDimensions()`, `WeightSize()`, and +`SetWeights()`. -Since some of this methods return internal class members we have to define them. +```c++ +template +void ExampleLayer::ComputeOutputDimensions() +{ + // The output size is the same as the input size. + this->outputDimensions = this->inputDimensions; +} -@code -private: - size_t inSize, outSize; - OutputDataType weights, delta, gradient, outputParameter; - InputDataType inputParameter; -@endcode +template +size_t ExampleLayer::WeightSize() const +{ + size_t numWeights = this->inputDimensions[0]; + for (size_t i = 1; i < this->inputDimensions.size(); ++i) + numWeights *= this->inputDimensions[i]; + return numWeights; +} -Note some members are just here so \c ExampleLayer compiles without warning. -For instance, \c inputSize is not required to be a member of every type of -layer. +template +void ExampleLayer::SetWeights(typename MatType::elem_type* weightsPtr) +{ + MakeAlias(weights, weightsPtr, WeightSize(), 1); +} +``` -There is one last method that is especially interesting for a layer that shares -parameter. Since the layer weights are set once the complete model is defined, -it's not possible to split the weights during the construction time. To solve -this issue, a layer can implement the \c Reset() method which is called once the -layer parameter is set. +## Model Setup & Training -@section model_setup_training_anntut Model Setup & Training - -Once the base container is selected (\c FNN or \c RNN), the \c Add method can be +Once the base container is selected (`FNN` or `RNN`), the `Add` method can be used to add layers to the model. The code below adds two linear layers to the model---the first takes 512 units as input and gives 256 output units, and the second takes 256 units as input and gives 128 output units. -@code +```c++ FFN<> model; -model.Add >(512, 256); -model.Add >(256, 128); -@endcode +model.Add(256); +model.Add(128); +``` The model is trained on Armadillo matrices. For training a model, you will -typically use the \c Train() function: +typically use the `Train()` function: -@code +```c++ arma::mat trainingSet, trainingLabels; model.Train(trainingSet, trainingLabels); -@endcode +``` -You can use mlpack's \c Load() function to load a dataset like this: +You can use mlpack's `Load()` function to load a dataset like this: -@code +```c++ arma::mat trainingSet; data::Load("dataset.csv", dataset, true); -@endcode +``` -@code +```sh $ cat dataset.csv 0, 1, 4 1, 0, 5 1, 1, 1 2, 0, 2 -@endcode +``` The type does not necessarily need to be a CSV; it can be any supported storage format, assuming that it is a coordinate-format file in the format specified -above. For more information on mlpack file formats, see the documentation for -mlpack::data::Load(). +above. For more information on mlpack file formats, see the +[tutorial](../user/formats.md). -@note -It’s often a good idea to normalize or standardize your data, for example using: +*Note*: it’s often a good idea to normalize or standardize your data, for +example using: -@code +```c++ for (size_t i = 0; i < dataset.n_cols; ++i) dataset.col(i) /= norm(dataset.col(i), 2); -@endcode +``` Also, it is possible to retrain a model with new parameters or with a new reference set. This is functionally equivalent to creating a new model. -@section model_saving_loading_anntut Saving & Loading +## Saving & Loading -Using \c cereal (for more information about the internals see -[the Cereal website](http://uscilab.github.io/cereal/)), -mlpack is able to load and save machine learning models with ease. To save a -trained neural network to disk. The example below builds a model on the \c -thyroid dataset and then saves the model to the file \c model.xml for later use. +Using `cereal` (for more information about the internals see [the Cereal +website](http://uscilab.github.io/cereal/)), mlpack is able to load and save +machine learning models with ease. Note that due to the large compilation +overhead of enabling serialization, it is disabled by default. To enable +serialization for neural networks, define the `MLPACK_ENABLE_ANN_SERIALIZATION` +macro before including mlpack: -@code +```c++ +#define MLPACK_ENABLE_ANN_SERIALIZATION +#include +``` + +The example below builds a model on the `thyroid` dataset and then saves the +model to the file `model.xml` for later use. + +```c++ // Load the training set. arma::mat dataset; data::Load("thyroid_train.csv", dataset, true); @@ -607,14 +588,14 @@ arma::mat assignments; model.Predict(trainData, assignments); data::Save("model.xml", "model", model, false); -@endcode +``` -After this, the file model.xml will be available in the current working +After this, the file `model.xml` will be available in the current working directory. -Now, we can look at the output model file, \c model.xml: +Now, we can look at the output model file, `model.xml`: -@code +```sh $ cat model.xml @@ -686,24 +667,24 @@ $ cat model.xml -@endcode +``` -As you can see, the \c \ section of \c model.xml contains the trained +As you can see, the `` section of `model.xml` contains the trained network weights. We can see that this section also contains the network input size, which is 66 rows and 1 column. Note that in this example, we used three -different layers, as can be seen by looking at the \c \ section. Each +different layers, as can be seen by looking at the `` section. Each node has a unique id that is used to reconstruct the model when loading. -The models can also be saved as \c .bin or \c .txt; the \c .xml format provides +The models can also be saved as `.bin` or `.txt`; the `.xml` format provides a human-inspectable format (though the models tend to be quite complex and may be difficult to read). These models can then be re-used to be used for classification or other tasks. So, instead of saving or training a network, mlpack can also load a pre-trained -model. For instance, the example below will load the model from \c model.xml and -then generate the class predictions for the \c thyroid test dataset. +model. For instance, the example below will load the model from `model.xml` and +then generate the class predictions for the `thyroid` test dataset. -@code +```c++ data::Load("thyroid_test.csv", dataset, true); arma::mat testData = dataset.submat(0, 0, dataset.n_rows - 4, @@ -713,35 +694,34 @@ data::Load("model.xml", "model", model); arma::mat predictions; model.Predict(testData, predictions); -@endcode +``` This enables the possibility to distribute a model without having to train it first or simply to save a model for later use. Note that loading will also work on different machines. -@section extracting_parameters_anntut Extracting Parameters +## Extracting Parameters To access the weights from the neural network layers, you can call the following function on any initialized network: -@code +```c++ model.Parameters(); -@endcode +``` which will return the complete model parameters as an armadillo matrix object; however often it is useful to not only have the parameters for the complete network, but the parameters of a specific layer. The parameters for a specific -layer @c x can be accessed via the @c Parameters() member: +layer `x` can be accessed via the `Parameters()` member: -@code +```c++ arma::mat parametersX = model.Model()[x].Parameters(); -@endcode +``` -In the example above, we get the weights of the second layer. +In the example above, we get the weights of the `x`th layer. -@section further_anntut Further documentation +## Further documentation -For further documentation on the ann classes, consult the \ref mlpack::ann -"complete API documentation". - -*/ +For further documentation on the ann classes, consult the source code in the +`src/mlpack/methods/ann/` directory. Each of the layers are implemented in +`src/mlpack/methods/ann/layer`. diff --git a/doc/tutorials/approx_kfn/approx_kfn.txt b/doc/tutorials/approx_kfn.md similarity index 60% rename from doc/tutorials/approx_kfn/approx_kfn.txt rename to doc/tutorials/approx_kfn.md index be6f215ef2..39085b00b3 100644 --- a/doc/tutorials/approx_kfn/approx_kfn.txt +++ b/doc/tutorials/approx_kfn.md @@ -1,30 +1,25 @@ -/*! +# Approximate furthest neighbor search tutorial -@file approx_kfn.txt -@author Ryan Curtin -@brief Tutorial for how to use approximate furthest neighbor search in mlpack. +mlpack implements multiple strategies for approximate furthest neighbor +search in its `mlpack_approx_kfn` and `mlpack_kfn` command-line programs (each +program corresponds to different techniques). This tutorial discusses what +problems these algorithms solve and how to use each of the techniques that +mlpack implements. -@page akfntutorial Approximate furthest neighbor search (mlpack_approx_kfn) tutorial +Note that these functions are available as bindings to other languages too, and +all the examples here can be adapted accordingly. -@section intro_akfntut Introduction +mlpack implements five approximate furthest neighbor search algorithms: -\b mlpack implements multiple strategies for approximate furthest neighbor -search in its \c mlpack_approx_kfn and \c mlpack_kfn programs (each program -corresponds to different techniques). This tutorial discusses what problems -these algorithms solve and how to use each of the techniques that \b mlpack -implements. - -\b mlpack implements five approximate furthest neighbor search algorithms: - - - brute-force search (in \c mlpack_kfn) - - single-tree search (in \c mlpack_kfn) - - dual-tree search (in \c mlpack_kfn) - - query-dependent approximate furthest neighbor (QDAFN) (in \c mlpack_approx_kfn) - - DrusillaSelect (in \c mlpack_approx_kfn) + - brute-force search (in `mlpack_kfn`) + - single-tree search (in `mlpack_kfn`) + - dual-tree search (in `mlpack_kfn`) + - query-dependent approximate furthest neighbor (QDAFN) (in `mlpack_approx_kfn`) + - DrusillaSelect (in `mlpack_approx_kfn`) These methods are described in the following papers: -@code +``` @inproceedings{curtin2013tree, title={Tree-Independent Dual-Tree Algorithms}, author={Curtin, Ryan R. and March, William B. and Ram, Parikshit and Anderson, @@ -34,9 +29,9 @@ These methods are described in the following papers: pages={1435--1443}, year={2013} } -@endcode +``` -@code +``` @incollection{pagh2015approximate, title={Approximate furthest neighbor in high dimensions}, author={Pagh, Rasmus and Silvestri, Francesco and Sivertsen, Johan and Skala, @@ -46,9 +41,9 @@ These methods are described in the following papers: year={2015}, publisher={Springer} } -@endcode +``` -@code +``` @incollection{curtin2016fast, title={Fast approximate furthest neighbors with data-dependent candidate selection}, @@ -58,9 +53,9 @@ These methods are described in the following papers: year={2016}, publisher={Springer} } -@endcode +``` -@code +``` @article{curtin2018exploiting, title={Exploiting the structure of furthest neighbor search for fast approximate results}, @@ -69,109 +64,72 @@ These methods are described in the following papers: year={2018}, publisher={Elsevier} } -@endcode +``` The problem of furthest neighbor search is simple, and is the opposite of the much-more-studied nearest neighbor search problem. Given a set of reference -points \f$R\f$ (the set in which we are searching), and a set of query points -\f$Q\f$ (the set of points for which we want the furthest neighbor), our goal is -to return the \f$k\f$ furthest neighbors for each query point in \f$Q\f$: +points `R` (the set in which we are searching), and a set of query points `Q` +(the set of points for which we want the furthest neighbor), our goal is to +return the `k` furthest neighbors for each query point in `Q`: -\f[ -\operatorname{k-argmax}_{p_r \in R} d(p_q, p_r). -\f] +``` +k-argmax_{p_r in R} d(p_q, p_r). +``` -In order to solve this problem, \b mlpack provides a number of interfaces. +In order to solve this problem, mlpack provides a number of interfaces. - - two \ref cli_akfntut "simple command-line executables" to calculate - approximate furthest neighbors - - a simple \ref cpp_qdafn_akfntut "C++ class for QDAFN" - - a simple \ref cpp_ds_akfntut "C++ class for DrusillaSelect" - - a simple \ref cpp_ns_akfntut "C++ class for tree-based and brute-force" - search + - two simple command-line executables to calculate approximate furthest + neighbors + - a simple C++ class for QDAFN" + - a simple C++ class for DrusillaSelect + - a simple C++ class for tree-based and brute-force search -@section toc_akfntut Table of Contents +## Which algorithm should be used? -A list of all the sections this tutorial contains. - - - \ref intro_akfntut - - \ref toc_akfntut - - \ref which_akfntut - - \ref cli_akfntut - - \ref cli_ex1_akfntut - - \ref cli_ex2_akfntut - - \ref cli_ex3_akfntut - - \ref cli_ex4_akfntut - - \ref cli_ex5_akfntut - - \ref cli_ex6_akfntut - - \ref cli_ex7_akfntut - - \ref cli_ex8_akfntut - - \ref cli_final_akfntut - - \ref cpp_ds_akfntut - - \ref cpp_ex1_ds_akfntut - - \ref cpp_ex2_ds_akfntut - - \ref cpp_ex3_ds_akfntut - - \ref cpp_ex4_ds_akfntut - - \ref cpp_ex5_ds_akfntut - - \ref cpp_qdafn_akfntut - - \ref cpp_ex1_qdafn_akfntut - - \ref cpp_ex2_qdafn_akfntut - - \ref cpp_ex3_qdafn_akfntut - - \ref cpp_ex4_qdafn_akfntut - - \ref cpp_ex5_qdafn_akfntut - - \ref cpp_ns_akfntut - - \ref cpp_ex1_ns_akfntut - - \ref cpp_ex2_ns_akfntut - - \ref cpp_ex3_ns_akfntut - - \ref cpp_ex4_ns_akfntut - - \ref further_doc_akfntut - -@section which_akfntut Which algorithm should be used? - -There are three algorithms for furthest neighbor search that \b mlpack +There are three algorithms for furthest neighbor search that mlpack implements, and each is suited to a different setting. Below is some basic guidance on what should be used. Note that the question of "which algorithm should be used" is a very difficult question to answer, so the guidance below is just that---guidance---and may not be right for a particular problem. - - \c DrusillaSelect is very fast and will perform extremely well for datasets + - `DrusillaSelect` is very fast and will perform extremely well for datasets with outliers or datasets with structure (like low-dimensional datasets embedded in high dimensions) - - \c QDAFN is a random approach and therefore should be well-suited for + - `QDAFN` is a random approach and therefore should be well-suited for datasets with little to no structure - - The tree-based approaches (the \c KFN class and the \c mlpack_kfn program) is + - The tree-based approaches (the `KFN` class and the `mlpack_kfn` program) is best suited for low-dimensional datasets, and is most effective when very small levels of approximation are desired, or when exact results are desired. - Dual-tree search is most useful when the query set is large and structured (like for all-furthest-neighbor search). - Single-tree search is more useful when the query set is small. -@section cli_akfntut Command-line 'mlpack_approx_kfn' and 'mlpack_kfn' +## Command-line `mlpack_approx_kfn` and `mlpack_kfn` -\b mlpack provides two command-line programs to solve approximate furthest -neighbor search: +mlpack provides two command-line programs to solve approximate furthest neighbor +search: - - \c mlpack_approx_kfn, for the QDAFN and DrusillaSelect approaches - - \c mlpack_kfn, for exact and approximate tree-based approaches + - `mlpack_approx_kfn`, for the QDAFN and DrusillaSelect approaches + - `mlpack_kfn`, for exact and approximate tree-based approaches These two programs allow a large number of algorithms to be used to find -approximate furthest neighbors. Note that the \c mlpack_kfn program is also -documented by the \ref cli_nstut section of the \ref nstutorial page, as it -shares options with the \c mlpack_knn program. +approximate furthest neighbors. Note that the `mlpack_kfn` program is also +documented in the [KNN tutorial](knn.md) page, as it shares options with the +`mlpack_knn` program. -Below are several examples of how the \c mlpack_approx_kfn and \c mlpack_kfn -programs might be used. The first examples focus on the \c mlpack_approx_kfn -program, and the last few show how \c mlpack_kfn can be used to produce +Below are several examples of how the `mlpack_approx_kfn` and `mlpack_kfn` +programs might be used. The first examples focus on the `mlpack_approx_kfn` +program, and the last few show how `mlpack_kfn` can be used to produce approximate results. -@subsection cli_ex1_akfntut Calculate 5 furthest neighbors with default options +### Calculate 5 furthest neighbors with default options -Here we have a query dataset \c queries.csv and a reference dataset \c refs.csv +Here we have a query dataset `queries.csv` and a reference dataset `refs.csv` and we wish to find the 5 furthest neighbors of every query point in the -reference dataset. We may do that with the \c mlpack_approx_kfn algorithm, -using the default of the \c DrusillaSelect algorithm with default parameters. +reference dataset. We may do that with the `mlpack_approx_kfn` algorithm, +using the default of the `DrusillaSelect` algorithm with default parameters. -@code +```sh $ mlpack_approx_kfn -q queries.csv -r refs.csv -v -k 5 -n n.csv -d d.csv [INFO ] Loading 'refs.csv' as CSV data. Size is 3 x 1000. [INFO ] Building DrusillaSelect model... @@ -206,33 +164,32 @@ $ mlpack_approx_kfn -q queries.csv -r refs.csv -v -k 5 -n n.csv -d d.csv [INFO ] loading_data: 0.010689s [INFO ] saving_data: 0.005585s [INFO ] total_time: 0.018592s -@endcode +``` Convenient timers for parts of the program operation are printed. The results, -saved in \c n.csv and \c d.csv, indicate the furthest neighbors and distances -for each query point. The row of the output file indicates the query point that -the results are for. The neighbors are listed from furthest to nearest; so, the -4th element in the 3rd row of \c d.csv indicates the distance between the 3rd -query point in \c queries.csv and its approximate 4th furthest neighbor. -Similarly, the same element in \c n.csv indicates the index of the approximate -4th furthest neighbor (with respect to \c refs.csv). +saved in `n.csv` and `d.csv`, indicate the furthest neighbors and distances for +each query point. The row of the output file indicates the query point that the +results are for. The neighbors are listed from furthest to nearest; so, the 4th +element in the 3rd row of `d.csv` indicates the distance between the 3rd query +point in `queries.csv` and its approximate 4th furthest neighbor. Similarly, +the same element in `n.csv` indicates the index of the approximate 4th furthest +neighbor (with respect to `refs.csv`). -@subsection cli_ex2_akfntut Specifying algorithm parameters for DrusillaSelect +### Specifying algorithm parameters for `DrusillaSelect` -The \c -p (\c --num_projections) and \c -t (\c --num_tables) parameters affect -the running of the \c DrusillaSelect algorithm and the QDAFN algorithm. +The `-p` (`--num_projections`) and `-t` (`--num_tables`) parameters affect the +running of the `DrusillaSelect` algorithm and the QDAFN algorithm. Specifically, larger values for each of these parameters will search more possible candidate furthest neighbors and produce better results (at the cost of runtime). More details on how each of these parameters works is available in -the original papers, the \b mlpack source, or the documentation given by -\c --help. +the original papers, the mlpack source, or the documentation given by `--help`. -In the example below, we run \c DrusillaSelect to find 4 furthest neighbors -using 10 tables and 2 points in each table. In this case we have chosen to omit -the \c -n \c n.csv option, meaning that only the output candidate distances will -be written to \c d.csv. +In the example below, we run `DrusillaSelect` to find 4 furthest neighbors using +10 tables and 2 points in each table. In this case we have chosen to omit the +`-n n.csv` option, meaning that only the output candidate distances will be +written to `d.csv`. -@code +```sh $ mlpack_approx_kfn -q queries.csv -r refs.csv -v -k 4 -n n.csv -d d.csv -t 10 -p 2 [INFO ] Loading 'refs.csv' as CSV data. Size is 3 x 1000. [INFO ] Building DrusillaSelect model... @@ -267,17 +224,17 @@ $ mlpack_approx_kfn -q queries.csv -r refs.csv -v -k 4 -n n.csv -d d.csv -t 10 - [INFO ] loading_data: 0.008518s [INFO ] saving_data: 0.003734s [INFO ] total_time: 0.014019s -@endcode +``` -@subsection cli_ex3_akfntut Using QDAFN instead of DrusillaSelect +### Using QDAFN instead of `DrusillaSelect` The algorithm to be used for approximate furthest neighbor search can be -specified with the \c --algorithm (\c -a) option to the \c mlpack_approx_kfn +specified with the `--algorithm` (`-a`) option to the `mlpack_approx_kfn` program. Below, we use the QDAFN algorithm instead of the default. We leave -the \c -p and \c -t options at their defaults---even though QDAFN often requires +the `-p` and `-t` options at their defaults---even though QDAFN often requires more tables and points to get the same quality of results. -@code +```sh $ mlpack_approx_kfn -q queries.csv -r refs.csv -v -k 3 -n n.csv -d d.csv -a qdafn [INFO ] Loading 'refs.csv' as CSV data. Size is 3 x 1000. [INFO ] Building QDAFN model... @@ -312,17 +269,17 @@ $ mlpack_approx_kfn -q queries.csv -r refs.csv -v -k 3 -n n.csv -d d.csv -a qdaf [INFO ] qdafn_search: 0.000886s [INFO ] saving_data: 0.002253s [INFO ] total_time: 0.015465s -@endcode +``` -@subsection cli_ex4_akfntut Printing results quality with exact distances +### Printing results quality with exact distances -The \c mlpack_approx_kfn program can calculate the quality of the results if the -\c --calculate_error (\c -e) flag is specified. Below we use the program with -its default parameters and calculate the error, which is displayed in the -output. The error is only calculated for the furthest neighbor, not all k; -therefore, in this example we have set \c -k to \c 1. +The `mlpack_approx_kfn` program can calculate the quality of the results if the +`--calculate_error` (`-e`) flag is specified. Below we use the program with its +default parameters and calculate the error, which is displayed in the output. +The error is only calculated for the furthest neighbor, not all k; therefore, in +this example we have set `-k` to `1`. -@code +```sh $ mlpack_approx_kfn -q queries.csv -r refs.csv -v -k 1 -e -q -n n.csv [INFO ] Loading 'refs.csv' as CSV data. Size is 3 x 1000. [INFO ] Building DrusillaSelect model... @@ -363,33 +320,32 @@ $ mlpack_approx_kfn -q queries.csv -r refs.csv -v -k 1 -e -q -n n.csv [INFO ] loading_data: 0.008462s [INFO ] total_time: 0.011670s [INFO ] tree_building: 0.000202s -@endcode +``` Note that the output includes three lines indicating the error: -@code +```sh [INFO ] Average error: 1.08417. [INFO ] Maximum error: 1.28712. [INFO ] Minimum error: 1. -@endcode +``` In this case, a minimum error of 1 indicates an exact result, and over the entire query set the algorithm has returned a furthest neighbor candidate with maximum error 1.28712. -@subsection cli_ex5_akfntut Using cached exact distances for quality results +### Using cached exact distances for quality results However, for large datasets, calculating the error may take a long time, because the exact furthest neighbors must be calculated. Therefore, if the exact furthest neighbor distances are already known, they may be passed in with the -\c --exact_distances_file (\c -x) option in order to avoid the calculation. In -the example below, we assume \c exact.csv contains the exact furthest neighbor -distances. We run the \c qdafn algorithm in this example. +`--exact_distances_file` (`-x`) option in order to avoid the calculation. In +the example below, we assume `exact.csv` contains the exact furthest neighbor +distances. We run the `qdafn` algorithm in this example. -Note that the \c -e option must be specified for the \c -x option have any -effect. +Note that the `-e` option must be specified for the `-x` option have any effect. -@code +```sh $ mlpack_approx_kfn -q queries.csv -r refs.csv -k 1 -e -x exact.csv -n n.csv -v -a qdafn [INFO ] Loading 'refs.csv' as CSV data. Size is 3 x 1000. [INFO ] Building QDAFN model... @@ -427,18 +383,18 @@ $ mlpack_approx_kfn -q queries.csv -r refs.csv -k 1 -e -x exact.csv -n n.csv -v [INFO ] qdafn_search: 0.000793s [INFO ] saving_data: 0.000259s [INFO ] total_time: 0.012254s -@endcode +``` -@subsection cli_ex6_akfntut Using tree-based approximation with mlpack_kfn +### Using tree-based approximation with `mlpack_kfn` -The \c mlpack_kfn algorithm allows specifying a desired approximation level with -the \c --epsilon (\c -e) option. The parameter must be greater than or equal -to 0 and less than 1. A setting of 0 indicates exact search. +The `mlpack_kfn` algorithm allows specifying a desired approximation level with +the `--epsilon` (`-e`) option. The parameter must be greater than or equal to 0 +and less than 1. A setting of 0 indicates exact search. The example below runs dual-tree furthest neighbor search (the default algorithm) with the approximation parameter set to 0.5. -@code +```sh $ mlpack_kfn -q queries.csv -r refs.csv -v -k 3 -e 0.5 -n n.csv -d d.csv [INFO ] Loading 'refs.csv' as CSV data. Size is 3 x 1000. [INFO ] Loaded reference data from 'refs.csv' (3x1000). @@ -485,22 +441,22 @@ $ mlpack_kfn -q queries.csv -r refs.csv -v -k 3 -e 0.5 -n n.csv -d d.csv [INFO ] saving_data: 0.002850s [INFO ] total_time: 0.012667s [INFO ] tree_building: 0.000251s -@endcode +``` -Note that the format of the output files \c d.csv and \c n.csv are the same as -for \c mlpack_approx_kfn. +Note that the format of the output files `d.csv` and `n.csv` are the same as +for `mlpack_approx_kfn`. -@subsection cli_ex7_akfntut Different algorithms with 'mlpack_kfn' +### Different algorithms with `mlpack_kfn` -The \c mlpack_kfn program offers a large number of different algorithms that can -be used. The \c --algorithm (\c -a) may be used to specify three main different -algorithm types: \c naive (brute-force search), \c single_tree (single-tree -search), \c dual_tree (dual-tree search, the default), and \c greedy +The `mlpack_kfn` program offers a large number of different algorithms that can +be used. The `--algorithm` (`-a`) parameter may be used to specify three main +different algorithm types: `naive` (brute-force search), `single_tree` +(single-tree search), `dual_tree` (dual-tree search, the default), and `greedy` ("defeatist" greedy search, which goes to one leaf node of the tree then terminates). The example below uses single-tree search to find approximate neighbors with epsilon set to 0.1. -@code +```sh mlpack_kfn -q queries.csv -r refs.csv -v -k 3 -e 0.1 -n n.csv -d d.csv -a single_tree [INFO ] Loading 'refs.csv' as CSV data. Size is 3 x 1000. [INFO ] Loaded reference data from 'refs.csv' (3x1000). @@ -545,22 +501,22 @@ mlpack_kfn -q queries.csv -r refs.csv -v -k 3 -e 0.1 -n n.csv -d d.csv -a single [INFO ] saving_data: 0.003445s [INFO ] total_time: 0.013084s [INFO ] tree_building: 0.000250s -@endcode +``` -@subsection cli_ex8_akfntut Saving a model for later use +## Saving a model for later use -The \c mlpack_approx_kfn and \c mlpack_kfn programs both allow models to be -saved and loaded for future use. The \c --output_model_file (\c -M) option -allows specifying where to save a model, and the \c --input_model_file (\c -m) -option allows a model to be loaded instead of trained. So, if you specify -\c --input_model_file then you do not need to specify \c --reference_file -(\c -r), \c --num_projections (\c -p), or \c --num_tables (\c -t). +The `mlpack_approx_kfn` and `mlpack_kfn` programs both allow models to be saved +and loaded for future use. The `--output_model_file` (`-M`) option allows +specifying where to save a model, and the `--input_model_file` (`-m`) option +allows a model to be loaded instead of trained. So, if you specify +`--input_model_file` then you do not need to specify `--reference_file` (`-r`), +`--num_projections` (`-p`), or `--num_tables` (`-t`). The example below saves a model with 10 projections and 5 tables. Note that -neither \c --query_file (\c -q) nor \c -k are specified; this run only builds -the model and saves it to \c model.bin. +neither `--query_file` (`-q`) nor `-k` are specified; this run only builds the +model and saves it to `model.bin`. -@code +```sh $ mlpack_approx_kfn -r refs.csv -t 5 -p 10 -v -M model.bin [INFO ] Loading 'refs.csv' as CSV data. Size is 3 x 1000. [INFO ] Building DrusillaSelect model... @@ -588,12 +544,12 @@ $ mlpack_approx_kfn -r refs.csv -t 5 -p 10 -v -M model.bin [INFO ] drusilla_select_construct: 0.000321s [INFO ] loading_data: 0.004700s [INFO ] total_time: 0.007320s -@endcode +``` Now, with the model saved, we can run approximate furthest neighbor search on a query set using the saved model: -@code +```sh $ mlpack_approx_kfn -m model.bin -q queries.csv -k 3 -d d.csv -n n.csv -v [INFO ] Loading 'queries.csv' as CSV data. Size is 3 x 1000. [INFO ] Searching for 3 furthest neighbors with DrusillaSelect... @@ -624,35 +580,35 @@ $ mlpack_approx_kfn -m model.bin -q queries.csv -k 3 -d d.csv -n n.csv -v [INFO ] loading_data: 0.004599s [INFO ] saving_data: 0.003006s [INFO ] total_time: 0.009234s -@endcode +``` -These options work in the same way for both the \c mlpack_approx_kfn and -\c mlpack_kfn programs. +These options work in the same way for both the `mlpack_approx_kfn` and +`mlpack_kfn` programs. -@subsection cli_final_akfntut Final command-line program notes +### Final command-line program notes -Both the \c mlpack_kfn and \c mlpack_approx_kfn programs contain numerous -options not fully documented in these short examples. You can run each program -with the \c --help (\c -h) option for more information. +Both the `mlpack_kfn` and `mlpack_approx_kfn` programs contain numerous options +not fully documented in these short examples. You can run each program with the +`--help` (`-h`) option for more information. -@section cpp_ds_akfntut DrusillaSelect C++ class +## `DrusillaSelect` C++ class -\b mlpack provides a simple \c DrusillaSelect C++ class that can be used inside -of C++ programs to perform approximate furthest neighbor search. The class has -only one template parameter---\c MatType---which specifies the type of matrix to +mlpack provides a simple `DrusillaSelect` C++ class that can be used inside of +C++ programs to perform approximate furthest neighbor search. The class has +only one template parameter---`MatType`---which specifies the type of matrix to be use. That means the class can be used with either dense data (of type -\c arma::mat) or sparse data (of type \c arma::sp_mat). +`arma::mat`) or sparse data (of type `arma::sp_mat`). The following examples show simple usage of this class. -@subsection cpp_ex1_ds_akfntut Approximate furthest neighbors with defaults +### Approximate furthest neighbors with defaults -The code below builds a \c DrusillaSelect model with default options on the -matrix \c dataset, then queries for the approximate furthest neighbor of every -point in the \c queries matrix. +The code below builds a `DrusillaSelect` model with default options on the +matrix `dataset`, then queries for the approximate furthest neighbor of every +point in the `queries` matrix. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -668,21 +624,21 @@ DrusillaSelect<> ds(dataset); arma::mat distances; arma::Mat neighbors; ds.Search(queries, 1, neighbors, distances); -@endcode +``` -At the end of this code, both the \c distances and \c neighbors matrices will -have number of columns equal to the number of columns in the \c queries matrix. -So, each column of the \c distances and \c neighbors matrices are the distances -or neighbors of the corresponding column in the \c queries matrix. +At the end of this code, both the `distances` and `neighbors` matrices will have +number of columns equal to the number of columns in the `queries` matrix. So, +each column of the `distances` and `neighbors` matrices are the distances or +neighbors of the corresponding column in the `queries` matrix. -@subsection cpp_ex2_ds_akfntut Custom numbers of tables and projections +### Custom numbers of tables and projections -The following example constructs a DrusillaSelect model with 10 tables and 5 +The following example constructs a `DrusillaSelect` model with 10 tables and 5 projections. Once that is done it performs the same task as the previous example. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -698,17 +654,17 @@ DrusillaSelect<> ds(dataset, 10, 5); arma::mat distances; arma::Mat neighbors; ds.Search(queries, 1, neighbors, distances); -@endcode +``` -@subsection cpp_ex3_ds_akfntut Accessing the candidate set +### Accessing the candidate set -The \c DrusillaSelect algorithm merely scans the reference set and extracts a +The `DrusillaSelect` algorithm merely scans the reference set and extracts a number of points that will be queried in a brute-force fashion when the -\c Search() method is called. We can access this set with the \c CandidateSet() +`Search()` method is called. We can access this set with the `CandidateSet()` method. The code below prints the fifth point of the candidate set. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -720,18 +676,18 @@ DrusillaSelect<> ds(dataset, 10, 5); // Print the fifth point of the candidate set. std::cout << ds.CandidateSet().col(4).t(); -@endcode +``` -@subsection cpp_ex4_ds_akfntut Retraining on a new reference set +### Retraining on a new reference set -It is possible to retrain a \c DrusillaSelect model with new parameters or with -a new reference set. This is functionally equivalent to creating a new model. +It is possible to retrain a `DrusillaSelect` model with new parameters or with a +new reference set. This is functionally equivalent to creating a new model. The example code below creates a first \c DrusillaSelect model using 3 tables and 10 projections, and then retrains this with the same reference set using 10 tables and 3 projections. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -743,17 +699,17 @@ DrusillaSelect<> ds(dataset, 3, 10); // Now retrain with different parameters. ds.Train(dataset, 10, 3); -@endcode +``` -@subsection cpp_ex5_ds_akfntut Running on sparse data +### Running on sparse data -We can set the template parameter for \c DrusillaSelect to \c arma::sp_mat in +We can set the template parameter for `DrusillaSelect` to `arma::sp_mat` in order to perform furthest neighbor search on sparse data. This code below -creates a \c DrusillaSelect model using 4 tables and 6 projections with sparse +creates a `DrusillaSelect` model using 4 tables and 6 projections with sparse input data, then searches for 3 approximate furthest neighbors. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -769,26 +725,26 @@ DrusillaSelect ds(dataset, 4, 6); arma::Mat neighbors; arma::mat distances; ds.Search(querySet, 3, neighbors, distances); -@endcode +``` -@section cpp_qdafn_akfntut QDAFN C++ class +## QDAFN C++ class -\b mlpack also provides a standalone simple \c QDAFN class for furthest neighbor -search. The API for this class is virtually identical to the \c DrusillaSelect +mlpack also provides a standalone simple `QDAFN` class for furthest neighbor +search. The API for this class is virtually identical to the `DrusillaSelect` class, and also has one template parameter to specify the type of matrix to be used (dense or sparse or other). -The following subsections demonstrate usage of the \c QDAFN class in the same -way as the previous section's examples for \c DrusillaSelect. +The following subsections demonstrate usage of the `QDAFN` class in the same way +as the previous section's examples for `DrusillaSelect`. -@subsection cpp_ex1_qdafn_akfntut Approximate furthest neighbors with defaults +### Approximate furthest neighbors with defaults -The code below builds a \c QDAFN model with default options on the -matrix \c dataset, then queries for the approximate furthest neighbor of every -point in the \c queries matrix. +The code below builds a `QDAFN` model with default options on the matrix +`dataset`, then queries for the approximate furthest neighbor of every point in +the `queries` matrix. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -804,21 +760,21 @@ QDAFN<> qd(dataset); arma::mat distances; arma::Mat neighbors; qd.Search(queries, 1, neighbors, distances); -@endcode +``` -At the end of this code, both the \c distances and \c neighbors matrices will -have number of columns equal to the number of columns in the \c queries matrix. -So, each column of the \c distances and \c neighbors matrices are the distances -or neighbors of the corresponding column in the \c queries matrix. +At the end of this code, both the `distances` and `neighbors` matrices will have +number of columns equal to the number of columns in the `queries` matrix. So, +each column of the `distances` and `neighbors` matrices are the distances or +neighbors of the corresponding column in the `queries` matrix. -@subsection cpp_ex2_qdafn_akfntut Custom numbers of tables and projections +### Custom numbers of tables and projections -The following example constructs a QDAFN model with 15 tables and 30 +The following example constructs a `QDAFN` model with 15 tables and 30 projections. Once that is done it performs the same task as the previous example. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -834,18 +790,18 @@ QDAFN<> qdafn(dataset, 15, 30); arma::mat distances; arma::Mat neighbors; qdafn.Search(queries, 1, neighbors, distances); -@endcode +``` -@subsection cpp_ex3_qdafn_akfntut Accessing the candidate set +### Accessing the candidate set -The \c QDAFN algorithm scans the reference set, extracting points that have been +The `QDAFN` algorithm scans the reference set, extracting points that have been projected onto random directions. Each random direction corresponds to a single -table. The \c QDAFN class stores these points as a vector of matrices, which -can be accessed with the \c CandidateSet() method. The code below prints the -fifth point of the candidate set of the third table. +table. The `QDAFN` class stores these points as a vector of matrices, which can +be accessed with the `CandidateSet()` method. The code below prints the fifth +point of the candidate set of the third table. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -857,18 +813,18 @@ QDAFN<> qdafn(dataset, 10, 5); // Print the fifth point of the candidate set. std::cout << ds.CandidateSet(2).col(4).t(); -@endcode +``` -@subsection cpp_ex4_qdafn_akfntut Retraining on a new reference set +### Retraining on a new reference set -It is possible to retrain a \c QDAFN model with new parameters or with -a new reference set. This is functionally equivalent to creating a new model. -The example code below creates a first \c QDAFN model using 10 tables -and 40 projections, and then retrains this with the same reference set using 15 -tables and 25 projections. +It is possible to retrain a `QDAFN` model with new parameters or with a new +reference set. This is functionally equivalent to creating a new model. The +example code below creates a first `QDAFN` model using 10 tables and 40 +projections, and then retrains this with the same reference set using 15 tables +and 25 projections. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -880,17 +836,17 @@ QDAFN<> qdafn(dataset, 3, 10); // Now retrain with different parameters. qdafn.Train(dataset, 10, 3); -@endcode +``` -@subsection cpp_ex5_qdafn_akfntut Running on sparse data +### Running on sparse data -We can set the template parameter for \c QDAFN to \c arma::sp_mat in -order to perform furthest neighbor search on sparse data. This code below -creates a \c QDAFN model using 20 tables and 60 projections with sparse -input data, then searches for 3 approximate furthest neighbors. +We can set the template parameter for `QDAFN` to `arma::sp_mat` in order to +perform furthest neighbor search on sparse data. This code below creates a +`QDAFN` model using 20 tables and 60 projections with sparse input data, then +searches for 3 approximate furthest neighbors. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -906,30 +862,30 @@ QDAFN qdafn(dataset, 20, 60); arma::Mat neighbors; arma::mat distances; qdafn.Search(querySet, 3, neighbors, distances); -@endcode +``` -@section cpp_ns_akfntut KFN C++ class +## KFN C++ class -The extensive \c NeighborSearch class also provides a way to search for +The extensive `NeighborSearch` class also provides a way to search for approximate furthest neighbors using a different, tree-based technique. For -full documentation on this class, see the -\ref nstutorial "NeighborSearch tutorial". The \c KFN class is a convenient -typedef of the \c NeighborSearch class that can be used to perform the furthest -neighbors task with kd-trees. +full documentation on this class, see the [NeighborSearch +tutorial](nstutorial.md). The `KFN` class is a convenient typedef of the +`NeighborSearch` class that can be used to perform the furthest neighbors task +with `kd`-trees. -In the following subsections, the \c KFN class is used in short code examples. +In the following subsections, the `KFN` class is used in short code examples. -@subsection cpp_ex1_ns_akfntut Simple furthest neighbors example +### Simple furthest neighbors example -The \c KFN class has construction semantics similar to \c DrusillaSelect and -\c QDAFN. The example below constructs a \c KFN object (which will build the +The `KFN` class has construction semantics similar to `DrusillaSelect` and +`QDAFN`. The example below constructs a `KFN` object (which will build the tree on the reference set), but note that the third parameter to the constructor allows us to specify our desired level of approximation. In this example we -choose epsilon = 0.05. Then, the code searches for 3 approximate furthest +choose `epsilon = 0.05`. Then, the code searches for 3 approximate furthest neighbors. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -946,15 +902,15 @@ KFN kfn(dataset, KFN::DUAL_TREE_MODE, 0.05); arma::Mat neighbors; arma::mat distances; kfn.Search(querySet, 3, neighbors, distances); -@endcode +``` -@subsection cpp_ex2_ns_akfntut Retraining on a new reference set +### Retraining on a new reference set -Like the \c QDAFN and \c DrusillaSelect classes, the \c KFN class is capable of +Like the `QDAFN` and `DrusillaSelect` classes, the `KFN` class is capable of retraining on a new reference set. The code below demonstrates this. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -968,16 +924,16 @@ KFN kfn(dataset, DUAL_TREE_MODE, 0.1); // Retrain on the new reference set. kfn.Train(newDataset); -@endcode +``` -@subsection cpp_ex3_ns_akfntut Searching in single-tree mode +### Searching in single-tree mode The particular mode to be used in search can be specified in the constructor. In this example, we use single-tree search (as opposed to the default of dual-tree search). -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -994,17 +950,17 @@ KFN kfn(dataset, SINGLE_TREE_MODE, 0.25); arma::Mat neighbors; arma::mat distances; kfn.Search(querySet, 5, neighbors, distances); -@endcode +``` -@subsection cpp_ex4_ns_akfntut Searching in brute-force mode +### Searching in brute-force mode If desired, brute-force search ("naive search") can be used to find the furthest neighbors; however, the result will not be approximate---it will be exact (since every possibility will be considered). The code below performs exact furthest -neighbor search by using the \c KFN class in brute-force mode. +neighbor search by using the `KFN` class in brute-force mode. -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -1021,16 +977,11 @@ KFN kfn(dataset, NAIVE_MODE); arma::Mat neighbors; arma::mat distances; kfn.Search(querySet, 2, neighbors, distances); -@endcode +``` -@section further_doc_akfntut Further documentation +## Further documentation For further documentation on the approximate furthest neighbor facilities -offered by \b mlpack, consult the following documentation: - - - \ref nstutorial - - \ref mlpack::neighbor::QDAFN "QDAFN class documentation" - - \ref mlpack::neighbor::DrusillaSelect "DrusillaSelect class documentation" - - \ref mlpack::neighbor::NeighborSearch "NeighborSearch class documentation" - -*/ +offered by mlpack, see also [the NeighborSearch tutorial](nstutorial.md). Also, +each class (`QDAFN`, `DrusillaSelect`, `NeighborSelect`) are well-documented, +and more details can be found in the source code documentation. diff --git a/doc/tutorials/cf.md b/doc/tutorials/cf.md new file mode 100644 index 0000000000..c2bf4b48d1 --- /dev/null +++ b/doc/tutorials/cf.md @@ -0,0 +1,438 @@ +# Collaborative Filtering Tutorial + +Collaborative filtering is an increasingly popular approach for recommender +systems. A typical formulation of the problem is as follows: there are `n` +users and `m` items, and each user has rated some of the items. We want to +provide each user with a recommendation for an item they have not rated yet, +which they are likely to rate highly. In another formulation, we may want to +predict a user's rating of an item. This type of problem has been considered +extensively, especially in the context of the Netflix prize. The winning +approach for the Netflix prize was a collaborative filtering approach which +utilized matrix decomposition. More information on their approach can be found +in the following paper: + +``` +@article{koren2009matrix, + title={Matrix factorization techniques for recommender systems}, + author={Koren, Yehuda and Bell, Robert and Volinsky, Chris}, + journal={Computer}, + number={8}, + pages={30--37}, + year={2009}, + publisher={IEEE} +} +``` + +The key to this approach is that the data is represented as an incomplete matrix +`V` with size `n x m`, where `V_ij` represents user `i`'s rating of item `j`, if +that rating exists. The task, then, is to complete the entries of the matrix. + +In the matrix factorization framework, the matrix `V` is assumed to be low-rank +and decomposed into components as `V ~ WH` according to some heuristic. + +In order to solve problems of this form, mlpack provides both an easy-to-use +binding (detailed here as a command-line program), and a simple yet flexible C++ +API that allows the implementation of new collaborative filtering techniques. + +## The `mlpack_cf` program + +mlpack provides a command-line program, `mlpack_cf`, which is used to perform +collaborative filtering on a given dataset. It can provide neighborhood-based +recommendations for users. The algorithm used for matrix factorization is +configurable, and the parameters of each algorithm are also configurable. *Note +that mlpack also provides the `cf()` function in other languages too; however, +this tutorial focuses on the command-line program `mlpack_cr`. It is easy to +adapt each example to each other language, though.* + +The following examples detail usage of the `mlpack_cf` program. Note that you +can get documentation on all the possible parameters by typing: + +```sh +$ mlpack_cf --help +``` + +### Input format for `mlpack_cf` + +The input file for the `mlpack_cf` program is specified with the +`--training_file` or `-t` option. This file is a coordinate-format sparse +matrix, similar to the Matrix Market (MM) format. The first coordinate is the +user id; the second coordinate is the item id; and the third coordinate is the +rating. So, for instance, a dataset with 3 users and 2 items, and ratings +between 1 and 5, might look like the following: + +```sh +$ cat dataset.csv +0, 1, 4 +1, 0, 5 +1, 1, 1 +2, 0, 2 +``` + +This dataset has four ratings: user 0 has rated item 1 with a rating of 4; user +1 has rated item 0 with a rating of 5; user 1 has rated item 1 with a rating of +1; and user 2 has rated item 0 with a rating of 2. Note that the user and item +indices start from 0, and the identifiers must be numeric indices, and not +names. + +The type does not necessarily need to be a csv; it can be any supported storage +format, assuming that it is a coordinate-format file in the format specified +above. For more information on mlpack file formats, see the documentation for +`mlpack::data::Load()`. + +### `mlpack_cf` with default parameters + +In this example, we have a dataset from MovieLens, and we want to use +`mlpack_cf` with the default parameters, which will provide 5 recommendations +for each user, and we wish to save the results in the file +`recommendations.csv`. Assuming that our dataset is in the file +`MovieLens-100k.csv` and it is in the correct format, we may use the `mlpack_cf` +executable as below: + +```sh +$ mlpack_cf -t MovieLens-100k.csv -v -o recommendations.csv +``` + +The `-v` option provides verbose output, and may be omitted if desired. Now, +for each user, we have recommendations in `recommendations.csv`: + +```sh +$ head recommendations.csv +317,422,482,356,495 +116,120,180,6,327 +312,49,116,99,236 +312,116,99,236,285 +55,190,317,194,63 +171,209,180,175,95 +208,0,94,87,57 +99,97,0,203,172 +257,99,180,287,0 +171,203,172,209,88 +``` + +So, for user 0, the top 5 recommended items that user 0 has not rated are items +317, 422, 482, 356, and 495. For user 5, the recommendations are on the sixth +line: 171, 209, 180, 175, 95. + +The `mlpack_cf` program can be built into a larger recommendation framework, +with a preprocessing step that can turn user information and item information +into numeric IDs, and a postprocessing step that can map these numeric IDs back +to the original information. + +### Saving `mlpack_cf` models + +The `mlpack_cf` program is able to save a particular model for later loading. +Saving a model can be done with the `--output_model_file` or `-M` option. The +example below builds a CF model on the `MovieLens-100k.csv` dataset, and then +saves the model to the file `cf-model.xml` for later usage. + +```sh +$ mlpack_cf -t MovieLens-100k.csv -M cf-model.xml -v +``` + +The models can also be saved as `.bin` or `.txt`; the `.xml` format provides +a human-inspectable format (though the models tend to be quite complex and may +be difficult to read). These models can then be re-used to provide specific +recommendations for certain users, or other tasks. + +### Loading `mlpack_cf` models + +Instead of training a model, the `mlpack_cf` model can also load a model to +provide recommendations, using the `--input_model_file` or `-m` option. For +instance, the example below will load the model from `cf-model.xml` and then +generate 3 recommendations for each user in the dataset, saving the results to +`recommendations.csv`. + +```sh +$ mlpack_cf -m cf-model.xml -v -o recommendations.csv +``` + +### Specifying rank of `mlpack_cf` decomposition + +By default, the matrix factorizations in the `mlpack_cf` program decompose the +data matrix into two matrices `W` and `H` with rank two. Often, this +default parameter is not correct, and it makes sense to use a higher-rank +decomposition. The rank can be specified with the `--rank` or `-R` parameter: + +```sh +$ mlpack_cf -t MovieLens-100k.csv -R 10 -v +``` + +In the example above, the data matrix will be decomposed into two matrices of +rank 10. In general, higher-rank decompositions will take longer, but will give +more accurate predictions. + +### `mlpack_cf` with single-user recommendation + +In the previous two examples, the output file `recommendations.csv` contains +one line for each user in the input dataset. But often, recommendations may +only be desired for a few users. In that case, we can assemble a file of query +users, with one user per line: + +```sh +$ cat query.csv +0 +17 +31 +``` + +Now, if we run the `mlpack_cf` executable with this query file, we will obtain +recommendations for users 0, 17, and 31: + +```sh +$ mlpack_cf -i MovieLens-100k.csv -R 10 -q query.csv -o recommendations.csv +$ cat recommendations.csv +474,356,317,432,473 +510,172,204,483,182 +0,120,236,257,126 +``` + +### `mlpack_cf` with non-default factorizer + +The `--algorithm` (or `-a`) parameter controls the factorizer that is used. +Several options are available: + + - `NMF`: non-negative matrix factorization; see `mlpack::amf::AMF` + - `SVDBatch`: SVD batch factorization + - `SVDIncompleteIncremental`: incomplete incremental SVD + - `SVDCompleteIncremental`: complete incremental SVD + - `RegSVD`: regularized SVD; see `mlpack::svd::RegularizedSVD` + +The default factorizer is `NMF`. The example below uses the `RegSVD` +factorizer: + +```sh +$ mlpack_cf -i MovieLens-100k.csv -R 10 -q query.csv -a RegSVD -o recommendations.csv +``` + +### `mlpack_cf` with non-default neighborhood size + +The `mlpack_cf` program produces recommendations using a neighborhood: similar +users in the query user's neighborhood will be averaged to produce predictions. +The size of this neighborhood is controlled with the `--neighborhood` (or `-n`) +option. An example using a neighborhood with 10 similar users is below: + +```sh +$ mlpack_cf -i MovieLens-100k.csv -R 10 -q query.csv -a RegSVD -n 10 +``` + +## The `CF` class + +The `CF` class in mlpack offers a simple, flexible API for performing +collaborative filtering for recommender systems within C++ applications. In the +constructor, the `CF` class takes a coordinate-list dataset and decomposes the +matrix according to the specified `FactorizerType` template parameter. + +Then, the `GetRecommendations()` function may be called to obtain +recommendations for certain users (or all users), and the `W()` and `H()` +matrices may be accessed to perform other computations. + +The data which the `CF` constructor takes should be an Armadillo matrix +(`arma::mat`) with three rows. The first row corresponds to users; the second +row corresponds to items; the third column corresponds to the rating. This is a +coordinate list format, like the format the `mlpack_cf` executable takes. The +`data::Load()` function can be used to load data. + +The following examples detail a few ways that the `CF` class can be used. + +### `CF` with default parameters + +This example constructs the `CF` object with default parameters and obtains +recommendations for each user, storing the output in the `recommendations` +matrix. + +```c++ +#include + +using namespace mlpack::cf; + +// The coordinate list of ratings that we have. +extern arma::mat data; +// The size of the neighborhood to use to get recommendations. +extern size_t neighborhood; +// The rank of the decomposition. +extern size_t rank; + +// Build the CF object and perform the decomposition. +// The constructor takes a default-constructed factorizer, which, by default, +// is of type amf::NMFALSFactorizer. +CF cf(data, amf::NMFALSFactorizer(), neighborhood, rank); + +// Store the results in this object. +arma::Mat recommendations; + +// Get 5 recommendations for all users. +cf.GetRecommendations(5, recommendations); +``` + +### `CF` with other factorizers + +mlpack provides a number of existing factorizers which can be used in place of +the default `mlpack::amf::NMFALSFactorizer` (which is non-negative matrix +factorization with alternating least squares update rules). These include: + + - `mlpack::amf::SVDBatchFactorizer` + - `mlpack::amf::SVDCompleteIncrementalFactorizer` + - `mlpack::amf::SVDIncompleteIncrementalFactorizer` + - `mlpack::amf::NMFALSFactorizer` + - `mlpack::svd::RegularizedSVD` + - `mlpack::svd::QUIC_SVD` + +The `amf::AMF` class has many other possibilities than those listed here; it is +a framework for alternating matrix factorization techniques. See the +`mlpack::amf::AMF` class documentation or [tutorial on AMF](amf.md) for more +information. + +The use of another factorizer is straightforward; the example from the previous +section is adapted below to use `svd::RegularizedSVD`: + +```c++ +#include + +using namespace mlpack::cf; + +// The coordinate list of ratings that we have. +extern arma::mat data; +// The size of the neighborhood to use to get recommendations. +extern size_t neighborhood; +// The rank of the decomposition. +extern size_t rank; + +// Build the CF object and perform the decomposition. +CF cf(data, svd::RegularizedSVD(), neighborhood, rank); + +// Store the results in this object. +arma::Mat recommendations; + +// Get 5 recommendations for all users. +cf.GetRecommendations(5, recommendations); +``` + +### Predicting individual user/item ratings + +The `Predict()` method can be used to predict the rating of an item by a certain +user, using the same neighborhood-based approach as the `GetRecommendations()` +function or the `mlpack_cf` executable. Below is an example of the use of that +function. + +The example below will obtain the predicted rating for item 50 by user 12. + +```c++ +#include + +using namespace mlpack::cf; + +// The coordinate list of ratings that we have. +extern arma::mat data; +// The size of the neighborhood to use to get recommendations. +extern size_t neighborhood; +// The rank of the decomposition. +extern size_t rank; + +// Build the CF object and perform the decomposition. +// The constructor takes a default-constructed factorizer, which, by default, +// is of type amf::NMFALSFactorizer. +CF cf(data, amf::NMFALSFactorizer(), neighborhood, rank); + +const double prediction = cf.Predict(12, 50); // User 12, item 50. +``` + +### Other operations with the `W` and `H` matrices + +Sometimes, the raw decomposed `W` and `H` matrices can be useful. The example +below obtains these matrices, and multiplies them against each other to obtain a +reconstructed data matrix with no missing values. + +```c++ +#include + +using namespace mlpack::cf; + +// The coordinate list of ratings that we have. +extern arma::mat data; +// The size of the neighborhood to use to get recommendations. +extern size_t neighborhood; +// The rank of the decomposition. +extern size_t rank; + +// Build the CF object and perform the decomposition. +// The constructor takes a default-constructed factorizer, which, by default, +// is of type amf::NMFALSFactorizer. +CF cf(data, amf::NMFALSFactorizer(), neighborhood, rank); + +// References to W and H matrices. +const arma::mat& W = cf.W(); +const arma::mat& H = cf.H(); + +// Multiply the matrices together. +arma::mat reconstructed = W * H; +``` + +## Template parameters for the `CF` class + +The `CF` class takes the `FactorizerType` as a template parameter to some of +its constructors and to the `Train()` function. The `FactorizerType` class +defines the algorithm used for matrix factorization. There are a number of +existing factorizers that can be used in mlpack; these were detailed in the +'other factorizers' example of the previous section. + +The `FactorizerType` class must implement one of the two following methods: + + - `Apply(arma::mat& data, const size_t rank, arma::mat& W, arma::mat& + H);` + - `Apply(arma::sp_mat& data, const size_t rank, arma::mat& W, arma::mat& + H);` + +The difference between these two methods is whether `arma::mat` or +`arma::sp_mat` is used as input. If `arma::mat` is used, then the data matrix +is a coordinate list with three columns, as in the constructor to the `CF` +class. If `arma::sp_mat` is used, then a sparse matrix is passed with the +number of rows equal to the number of items and the number of columns equal to +the number of users, and each nonzero element in the matrix corresponds to a +non-missing rating. + +The method that the factorizer implements is specified via the \c +FactorizerTraits class, which is a template metaprogramming traits class: + +```c++ +template +struct FactorizerTraits +{ + /** + * If true, then the passed data matrix is used for factorizer.Apply(). + * Otherwise, it is modified into a form suitable for factorization. + */ + static const bool UsesCoordinateList = false; +}; +``` + +If `FactorizerTraits::UsesCoordinateList` is `true`, then `CF` +will try to call `Apply()` with an `arma::mat` object. Otherwise, `CF` will try +to call `Apply()` with an `arma::sp_mat` object. Specifying the value of +`UsesCoordinateList` is straightforward; provide this specialization of the +`FactorizerTraits` class: + +```c++ +template<> +struct FactorizerTraits +{ + static const bool UsesCoordinateList = true; // Set your value here. +}; +``` + +The `Apply()` function also takes a reference to the matrices `W` and `H`. +When the `Apply()` function returns, the input data matrix should be decomposed +into these two matrices. `W` should have number of rows equal to the number of +items and number of columns equal to the `rank` parameter, and `H` should have +number of rows equal to the `rank` parameter, and number of columns equal to +the number of users. + +The `mlpack::amf::AMF` class can be used as a base for factorizers that +alternate between updating `W` and updating `H`. A useful reference is the [AMF +tutorial](amf.md). + +## Further documentation + +Further documentation for the `CF` class may be found in the comments in the +source code of the files in `src/mlpack/methods/cf/`. In addition, more +information on the \c AMF class of factorizers may be found in the sources for +`mlpack::amf::AMF`, in `src/mlpack/methods/amf/`. diff --git a/doc/tutorials/cf/cf.txt b/doc/tutorials/cf/cf.txt deleted file mode 100644 index ffa7c7ded1..0000000000 --- a/doc/tutorials/cf/cf.txt +++ /dev/null @@ -1,472 +0,0 @@ -/*! - -@file cf.txt -@author Ryan Curtin -@brief Tutorial for how to use the CF class and program. - -@page cftutorial Collaborative filtering tutorial - -@section intro_cftut Introduction - -Collaborative filtering is an increasingly popular approach for recommender -systems. A typical formulation of the problem is as follows: there are \f$n\f$ -users and \f$m\f$ items, and each user has rated some of the items. We want to -provide each user with a recommendation for an item they have not rated yet, -which they are likely to rate highly. In another formulation, we may want to -predict a user's rating of an item. This type of problem has been considered -extensively, especially in the context of the Netflix prize. The winning -approach for the Netflix prize was a collaborative filtering approach which -utilized matrix decomposition. More information on their approach can be found -in the following paper: - -@code -@article{koren2009matrix, - title={Matrix factorization techniques for recommender systems}, - author={Koren, Yehuda and Bell, Robert and Volinsky, Chris}, - journal={Computer}, - number={8}, - pages={30--37}, - year={2009}, - publisher={IEEE} -} -@endcode - -The key to this approach is that the data is represented as an incomplete matrix -\f$V \in \Re^{n \times m}\f$, where \f$V_{ij}\f$ represents user \f$i\f$'s -rating of item \f$j\f$, if that rating exists. The task, then, is to complete -the entries of the matrix. - -In the matrix factorization framework, the matrix \f$V\f$ is assumed to be -low-rank and decomposed into components as \f$V \approx WH\f$ according to some -heuristic. - -In order to solve problems of this form, \b mlpack provides: - - - a \ref cli_cftut "simple command-line interface" to perform collaborative filtering - - a \ref cf_cftut "simple C++ interface" to perform collaborative filtering - - an \ref cpp_cftut "extensible C++ interface" for implementing new collaborative filtering techniques - -@section toc_cftut Table of Contents - - - \ref intro_cftut - - \ref toc_cftut - - \ref cli_cftut - - \ref cli_input_format - - \ref ex1_cf_cli - - \ref ex1a_cf_cli - - \ref ex1b_cf_cli - - \ref ex2_cf_cli - - \ref ex3_cf_cli - - \ref ex4_cf_cli - - \ref ex5_cf_cli - - \ref cf_cftut - - \ref ex1_cf_cpp - - \ref ex2_cf_cpp - - \ref ex3_cf_cpp - - \ref ex4_cf_cpp - - \ref cpp_cftut - - \ref further_doc_cftut - -@section cli_cftut The 'mlpack_cf' program - -\b mlpack provides a command-line program, \c mlpack_cf, which is used to -perform collaborative filtering on a given dataset. It can provide -neighborhood-based recommendations for users. The algorithm used for matrix -factorization is configurable, and the parameters of each algorithm are also -configurable. - -The following examples detail usage of the \c mlpack_cf program. Note that you -can get documentation on all the possible parameters by typing: - -@code -$ mlpack_cf --help -@endcode - -@subsection cli_input_format Input format for mlpack_cf - -The input file for the \c mlpack_cf program is specified with the \c ---training_file or \c -t option. This file is a coordinate-format sparse -matrix, similar to the Matrix Market (MM) format. The first coordinate is the -user id; the second coordinate is the item id; and the third coordinate is the -rating. So, for instance, a dataset with 3 users and 2 items, and ratings -between 1 and 5, might look like the following: - -@code -$ cat dataset.csv -0, 1, 4 -1, 0, 5 -1, 1, 1 -2, 0, 2 -@endcode - -This dataset has four ratings: user 0 has rated item 1 with a rating of 4; user -1 has rated item 0 with a rating of 5; user 1 has rated item 1 with a rating of -1; and user 2 has rated item 0 with a rating of 2. Note that the user and item -indices start from 0, and the identifiers must be numeric indices, and not -names. - -The type does not necessarily need to be a csv; it can be any supported storage -format, assuming that it is a coordinate-format file in the format specified -above. For more information on mlpack file formats, see the documentation for -mlpack::data::Load(). - -@subsection ex1_cf_cli mlpack_cf with default parameters - -In this example, we have a dataset from MovieLens, and we want to use -\c mlpack_cf with the default parameters, which will provide 5 recommendations -for each user, and we wish to save the results in the file -\c recommendations.csv. Assuming that our dataset is in the file -\c MovieLens-100k.csv and it is in the correct format, we may use the -\c mlpack_cf executable as below: - -@code -$ mlpack_cf -t MovieLens-100k.csv -v -o recommendations.csv -@endcode - -The \c -v option provides verbose output, and may be omitted if desired. Now, -for each user, we have recommendations in \c recommendations.csv: - -@code -$ head recommendations.csv -317,422,482,356,495 -116,120,180,6,327 -312,49,116,99,236 -312,116,99,236,285 -55,190,317,194,63 -171,209,180,175,95 -208,0,94,87,57 -99,97,0,203,172 -257,99,180,287,0 -171,203,172,209,88 -@endcode - -So, for user 0, the top 5 recommended items that user 0 has not rated are items -317, 422, 482, 356, and 495. For user 5, the recommendations are on the sixth -line: 171, 209, 180, 175, 95. - -The \c mlpack_cf program can be built into a larger recommendation framework, -with a preprocessing step that can turn user information and item information -into numeric IDs, and a postprocessing step that can map these numeric IDs back -to the original information. - -@subsection ex1a_cf_cli Saving mlpack_cf models - -The \c mlpack_cf program is able to save a particular model for later loading. -Saving a model can be done with the \c --output_model_file or \c -M option. The -example below builds a CF model on the \c MovieLens-100k.csv dataset, and then -saves the model to the file \c cf-model.xml for later usage. - -@code -$ mlpack_cf -t MovieLens-100k.csv -M cf-model.xml -v -@endcode - -The models can also be saved as \c .bin or \c .txt; the \c .xml format provides -a human-inspectable format (though the models tend to be quite complex and may -be difficult to read). These models can then be re-used to provide specific -recommendations for certain users, or other tasks. - -@subsection ex1b_cf_cli Loading mlpack_cf models - -Instead of training a model, the \c mlpack_cf model can also load a model to -provide recommendations, using the \c --input_model_file or \c -m option. For -instance, the example below will load the model from \c cf-model.xml and then -generate 3 recommendations for each user in the dataset, saving the results to -\c recommendations.csv. - -@code -$ mlpack_cf -m cf-model.xml -v -o recommendations.csv -@endcode - -@subsection ex2_cf_cli Specifying rank of mlpack_cf decomposition - -By default, the matrix factorizations in the \c mlpack_cf program decompose the -data matrix into two matrices \f$W\f$ and \f$H\f$ with rank two. Often, this -default parameter is not correct, and it makes sense to use a higher-rank -decomposition. The rank can be specified with the \c --rank or \c -R parameter: - -@code -$ mlpack_cf -t MovieLens-100k.csv -R 10 -v -@endcode - -In the example above, the data matrix will be decomposed into two matrices of -rank 10. In general, higher-rank decompositions will take longer, but will give -more accurate predictions. - -@subsection ex3_cf_cli mlpack_cf with single-user recommendation - -In the previous two examples, the output file \c recommendations.csv contains -one line for each user in the input dataset. But often, recommendations may -only be desired for a few users. In that case, we can assemble a file of query -users, with one user per line: - -@code -$ cat query.csv -0 -17 -31 -@endcode - -Now, if we run the \c mlpack_cf executable with this query file, we will obtain -recommendations for users 0, 17, and 31: - -@code -$ mlpack_cf -i MovieLens-100k.csv -R 10 -q query.csv -o recommendations.csv -$ cat recommendations.csv -474,356,317,432,473 -510,172,204,483,182 -0,120,236,257,126 -@endcode - -@subsection ex4_cf_cli mlpack_cf with non-default factorizer - -The \c --algorithm (or \c -a ) parameter controls the factorizer that is used. -Several options are available: - - - \c 'NMF': non-negative matrix factorization; see mlpack::amf::AMF<> - - \c 'SVDBatch': SVD batch factorization - - \c 'SVDIncompleteIncremental': incomplete incremental SVD - - \c 'SVDCompleteIncremental': complete incremental SVD - - \c 'RegSVD': regularized SVD; see mlpack::svd::RegularizedSVD - -The default factorizer is \c 'NMF'. The example below uses the 'RegSVD' -factorizer: - -@code -$ mlpack_cf -i MovieLens-100k.csv -R 10 -q query.csv -a RegSVD -o recommendations.csv -@endcode - -@subsection ex5_cf_cli mlpack_cf with non-default neighborhood size - -The \c mlpack_cf program produces recommendations using a neighborhood: similar -users in the query user's neighborhood will be averaged to produce predictions. -The size of this neighborhood is controlled with the \c --neighborhood (or \c -n -) option. An example using a neighborhood with 10 similar users is below: - -@code -$ mlpack_cf -i MovieLens-100k.csv -R 10 -q query.csv -a RegSVD -n 10 -@endcode - -@section cf_cftut The 'CF' class - -The \c CF class in \b mlpack offers a simple, flexible API for performing -collaborative filtering for recommender systems within C++ applications. In the -constructor, the \c CF class takes a coordinate-list dataset and decomposes the -matrix according to the specified \c FactorizerType template parameter. - -Then, the \c GetRecommendations() function may be called to obtain -recommendations for certain users (or all users), and the \c W() and \c H() -matrices may be accessed to perform other computations. - -The data which the \c CF constructor takes should be an Armadillo matrix (\c -arma::mat ) with three rows. The first row corresponds to users; the second -row corresponds to items; the third column corresponds to the rating. This is a -coordinate list format, like the format the \c cf executable takes. The -data::Load() function can be used to load data. - -The following examples detail a few ways that the \c CF class can be used. - -@subsection ex1_cf_cpp CF with default parameters - -This example constructs the \c CF object with default parameters and obtains -recommendations for each user, storing the output in the \c recommendations -matrix. - -@code -#include - -using namespace mlpack::cf; - -// The coordinate list of ratings that we have. -extern arma::mat data; -// The size of the neighborhood to use to get recommendations. -extern size_t neighborhood; -// The rank of the decomposition. -extern size_t rank; - -// Build the CF object and perform the decomposition. -// The constructor takes a default-constructed factorizer, which, by default, -// is of type amf::NMFALSFactorizer. -CF cf(data, amf::NMFALSFactorizer(), neighborhood, rank); - -// Store the results in this object. -arma::Mat recommendations; - -// Get 5 recommendations for all users. -cf.GetRecommendations(5, recommendations); -@endcode - -@subsection ex2_cf_cpp CF with other factorizers - -\b mlpack provides a number of existing factorizers which can be used in place -of the default mlpack::amf::NMFALSFactorizer (which is non-negative matrix -factorization with alternating least squares update rules). These include: - - - mlpack::amf::SVDBatchFactorizer - - mlpack::amf::SVDCompleteIncrementalFactorizer - - mlpack::amf::SVDIncompleteIncrementalFactorizer - - mlpack::amf::NMFALSFactorizer - - mlpack::svd::RegularizedSVD - - mlpack::svd::QUIC_SVD - -The amf::AMF<> class has many other possibilities than those listed here; it is -a framework for alternating matrix factorization techniques. See the -\ref mlpack::amf::AMF<> "class documentation" or \ref amftutorial "tutorial on AMF" for -more information. - -The use of another factorizer is straightforward; the example from the previous -section is adapted below to use svd::RegularizedSVD: - -@code -#include -#include - -using namespace mlpack::cf; - -// The coordinate list of ratings that we have. -extern arma::mat data; -// The size of the neighborhood to use to get recommendations. -extern size_t neighborhood; -// The rank of the decomposition. -extern size_t rank; - -// Build the CF object and perform the decomposition. -CF cf(data, svd::RegularizedSVD(), neighborhood, rank); - -// Store the results in this object. -arma::Mat recommendations; - -// Get 5 recommendations for all users. -cf.GetRecommendations(5, recommendations); -@endcode - -@subsection ex3_cf_cpp Predicting individual user/item ratings - -The \c Predict() method can be used to predict the rating of an item by a -certain user, using the same neighborhood-based approach as the -\c GetRecommendations() function or the \c cf executable. Below is an example -of the use of that function. - -The example below will obtain the predicted rating for item 50 by user 12. - -@code -#include - -using namespace mlpack::cf; - -// The coordinate list of ratings that we have. -extern arma::mat data; -// The size of the neighborhood to use to get recommendations. -extern size_t neighborhood; -// The rank of the decomposition. -extern size_t rank; - -// Build the CF object and perform the decomposition. -// The constructor takes a default-constructed factorizer, which, by default, -// is of type amf::NMFALSFactorizer. -CF cf(data, amf::NMFALSFactorizer(), neighborhood, rank); - -const double prediction = cf.Predict(12, 50); // User 12, item 50. -@endcode - -@subsection ex4_cf_cpp Other operations with the W and H matrices - -Sometimes, the raw decomposed W and H matrices can be useful. The example below -obtains these matrices, and multiplies them against each other to obtain a -reconstructed data matrix with no missing values. - -@code -#include - -using namespace mlpack::cf; - -// The coordinate list of ratings that we have. -extern arma::mat data; -// The size of the neighborhood to use to get recommendations. -extern size_t neighborhood; -// The rank of the decomposition. -extern size_t rank; - -// Build the CF object and perform the decomposition. -// The constructor takes a default-constructed factorizer, which, by default, -// is of type amf::NMFALSFactorizer. -CF cf(data, amf::NMFALSFactorizer(), neighborhood, rank); - -// References to W and H matrices. -const arma::mat& W = cf.W(); -const arma::mat& H = cf.H(); - -// Multiply the matrices together. -arma::mat reconstructed = W * H; -@endcode - -@section cpp_cftut Template parameters for the 'CF' class - -The \c CF class takes the \c FactorizerType as a template parameter to some of -its constructors and to the \c Train() function. The \c FactorizerType class -defines the algorithm used for matrix factorization. There are a number of -existing factorizers that can be used in \b mlpack; these were detailed in the -\ref ex2_cf_cpp "'other factorizers' example" of the previous section. - -The \c FactorizerType class must implement one of the two following methods: - - - Apply(arma::mat& data, const size_t rank, arma::mat& W, arma::mat& - H); - - Apply(arma::sp_mat& data, const size_t rank, arma::mat& W, arma::mat& - H); - -The difference between these two methods is whether \c arma::mat or \c -arma::sp_mat is used as input. If \c arma::mat is used, then the data matrix is -a coordinate list with three columns, as in the constructor to the \c CF class. -If \c arma::sp_mat is used, then a sparse matrix is passed with the number of -rows equal to the number of items and the number of columns equal to the number -of users, and each nonzero element in the matrix corresponds to a non-missing -rating. - -The method that the factorizer implements is specified via the \c -FactorizerTraits class, which is a template metaprogramming traits class: - -@code -template -struct FactorizerTraits -{ - /** - * If true, then the passed data matrix is used for factorizer.Apply(). - * Otherwise, it is modified into a form suitable for factorization. - */ - static const bool UsesCoordinateList = false; -}; -@endcode - -If \c FactorizerTraits::UsesCoordinateList is \c true, then \c CF -will try to call \c Apply() with an \c arma::mat object. Otherwise, \c CF will -try to call \c Apply() with an \c arma::sp_mat object. Specifying the value of -\c UsesCoordinateList is straightforward; provide this specialization of the -\c FactorizerTraits class: - -@code -template<> -struct FactorizerTraits -{ - static const bool UsesCoordinateList = true; // Set your value here. -}; -@endcode - -The \c Apply() function also takes a reference to the matrices \c W and \c H. -When the \c Apply() function returns, the input data matrix should be decomposed -into these two matrices. \c W should have number of rows equal to the number of -items and number of columns equal to the \c rank parameter, and \c H should have -number of rows equal to the \c rank parameter, and number of columns equal to -the number of users. - -The \ref mlpack::amf::AMF<> "amf::AMF<> class" can be used as a base for -factorizers that alternate between updating \c W and updating \c H. A useful -reference is the \ref amftutorial "AMF tutorial". - -@section further_doc_cftut Further documentation - -Further documentation for the \c CF class may be found in the \ref -mlpack::cf "complete API documentation". In addition, more information on -the \c AMF class of factorizers may be found in its \ref mlpack::amf::AMF<> -"complete API documentation". - -*/ diff --git a/doc/tutorials/data_loading/datasetmapper.txt b/doc/tutorials/data_loading/datasetmapper.txt deleted file mode 100644 index bbd16fb76a..0000000000 --- a/doc/tutorials/data_loading/datasetmapper.txt +++ /dev/null @@ -1,192 +0,0 @@ -/*! - -@file datasetmapper.txt -@author Gopi Tatiraju -@breif Introduction and tutorial for how to use DatasetMapper in mlpack. - -@page datasetmapper DatasetMapper Tutorial - -@section intro_datasetmapper Introduction - -DatasetMapper is a class which holds information about a dataset. This can be -used when dataset contains categorical non-numeric features which should be -mapped to numeric features. A simple example can be - -``` -7,5,True,3 -6,3,False,4 -4,8,False,2 -9,3,True,3 -``` - -The above dataset will be represented as - -``` -7,5,0,3 -6,3,1,4 -4,8,1,2 -9,3,0,3 -``` - -Here Mappings are - -- `True` mapped to `0` -- `False` mapped to `1` - -``` -**Note** DatasetMapper converts non-numeric values in the order -in which it encounters them in dataset. Therefore there is a chance that -`True` might get mapped to `0` if it encounters `True` before `False`. -This `0` and `1` are not to be confused with C++ bool notations. These -are mapping created by `mpack::DatasetMapper`. -``` - -DatasetMapper provides an easy API to load such data and stores all the -necessary information of the dataset. - -@section toc_datasetmapper Table of Contents - -A list of all sections - - - \ref intro_datasetmapper - - \ref toc_datasetmapper - - \ref load - - \ref dimensions - - \ref type - - \ref numofmappings - - \ref checkmappings - - \ref unmapstring - - \ref unmapvalue - -@section load Loading data - -To use \b DatasetMapper we have to call a specific overload of `data::Load()` -fucntion. - -@code -using namespace mlpack; - -arma::mat data; -data::DatasetMapper info; -data::Load("dataset.csv", data, info); -@endcode - -Dataset -``` -7, 5, True, 3 -6, 3, False, 4 -4, 8, False, 2 -9, 3, True, 3 -``` - -@section dimensions Dimensionality - -There are two ways to initialize a DatasetMapper object. - -* First is to initialize the object and set each property yourself. - -* Second is to pass the object to Load() in which case mlpack will populate -the object. If we use the latter option then the dimensionality will be same -as what's in the data file. - -@code -std::cout << info.Dimensionality(); -@endcode - -@code -4 -@endcode - -@section type Type of each Dimension - -Each dimension can be of either of the two types - - data::Datatype::numeric - - data::Datatype::categorical - -\c `Type(size_t dimension)` takes an argument dimension which is the row -number for which you want to know the type - -This will return an enum `data::Datatype`, which is casted to -`size_t` when we print them using `std::cout` - - 0 represents `data::Datatype::numeric` - - 1 represents `data::Datatype::categorical` - -@code -std::cout << info.Type(0) << "\n"; -std::cout << info.Type(1) << "\n"; -std::cout << info.Type(2) << "\n"; -std::cout << info.Type(3) << "\n"; -@endcode - -@code -0 -0 -1 -0 -@endcode - -@section numofmappings Number of Mappings - -If the type of a dimension is `data::Datatype::categorical`, then during -loading, each unique token in that dimension will be mapped to an integer -starting with 0. - -\b NumMappings(size_t dimension) takes dimension as an argument and returns the number of -mappings in that dimension, if the dimension is a number or there are no mappings then it -will return 0. - -@code -std::cout << info.NumMappings(0) << "\n"; -std::cout << info.NumMappings(1) << "\n"; -std::cout << info.NumMappings(2) << "\n"; -std::cout << info.NumMappings(3) << "\n"; -@endcode - -@code -0 -0 -2 -0 -@endcode - -@section checkmappings Check Mappings - -There are two ways to check the mappings. - - Enter the string to get mapped integer - - Enter the mapped integer to get string - -@subsection unmapstring UnmapString - -\b UnmapString(int value, size_t dimension, size_t unmappingIndex = 0UL) - - value is the integer for which you want to find the mapped value - - dimension is the dimension in which you want to check the mappings - -@code -std::cout << info.UnmapString(0, 2) << "\n"; -std::cout << info.UnmapString(1, 2) << "\n"; -@endcode - -@code -T -F -@endcode - -@subsection unmapvalue UnmapValue - -\b UnmapValue(const std::string &input, size_t dimension) - - input is the mapped value for which you want to find mapping - - dimension is the dimension in which you want to find the mapped value - -@code -std::cout << info.UnmapValue("T", 2) << "\n"; -std::cout << info.UnmapValue("F", 2) << "\n"; -@endcode - -@code -0 -1 -@endcode - -These are basic uses of DatasetMapper. Some advance use cases will be added soon. - -*/ diff --git a/doc/tutorials/datasetmapper.md b/doc/tutorials/datasetmapper.md new file mode 100644 index 0000000000..a841071216 --- /dev/null +++ b/doc/tutorials/datasetmapper.md @@ -0,0 +1,186 @@ +# DatasetMapper tutorial + +`DatasetMapper` is a class which holds information about a dataset. This can be +used when dataset contains categorical non-numeric features which should be +mapped to numeric features. A simple example can be + +``` +7,5,True,3 +6,3,False,4 +4,8,False,2 +9,3,True,3 +``` + +The above dataset will be represented as + +``` +7,5,0,3 +6,3,1,4 +4,8,1,2 +9,3,0,3 +``` + +Here the mappings are + +- `True` mapped to `0` +- `False` mapped to `1` + +**Note**: `DatasetMapper` converts non-numeric values in the order in which it +encounters them in the dataset. Therefore there is a chance that `True` might +get mapped to `0` if it encounters `True` before `False`. This `0` and `1` are +not to be confused with C++ `bool` notations. These are mapping created by +`mlpack::DatasetMapper`. + +`DatasetMapper` provides an easy API to load such data and stores all the +necessary information of the dataset. + +## Loading data + +To use `DatasetMapper` we have to call a specific overload of the `data::Load()` +function. + +```c++ +using namespace mlpack; + +arma::mat data; +data::DatasetMapper info; +data::Load("dataset.csv", data, info); +``` + +Dataset: + +``` +7, 5, True, 3 +6, 3, False, 4 +4, 8, False, 2 +9, 3, True, 3 +``` + +## Dimensionality + +There are two ways to initialize a DatasetMapper object. + +* The first is to initialize the object and set each property yourself. + +* The second is to pass the object to `Load()` without initialization, and + mlpack will populate the object. If we use the latter option then the + dimensionality will be same as what's in the data file. + +```c++ +std::cout << info.Dimensionality(); +``` + +``` +4 +``` + +## Type of each dimension + +Each dimension can be of either of the two types: + + - `data::Datatype::numeric` + - `data::Datatype::categorical` + +The function `Type(size_t dimension)` takes an argument dimension which is the +row number for which you want to know the type + +This will return an enum `data::Datatype`, which is cast to `size_t` when we +print them using `std::cout`. + + - `0` represents `data::Datatype::numeric` + - `1` represents `data::Datatype::categorical` + +```c++ +std::cout << info.Type(0) << "\n"; +std::cout << info.Type(1) << "\n"; +std::cout << info.Type(2) << "\n"; +std::cout << info.Type(3) << "\n"; +``` + +This produces: + +``` +0 +0 +1 +0 +``` + +## Number of mappings + +If the type of a dimension is `data::Datatype::categorical`, then during +loading, each unique token in that dimension will be mapped to an integer +starting with `0`. + +`NumMappings(size_t dimension)` takes `dimension` as an argument and returns the +number of mappings in that dimension, if the dimension is numeric, or there are +no mappings, then it will return 0. + +```c++ +std::cout << info.NumMappings(0) << "\n"; +std::cout << info.NumMappings(1) << "\n"; +std::cout << info.NumMappings(2) << "\n"; +std::cout << info.NumMappings(3) << "\n"; +``` + +will print: + +``` +0 +0 +2 +0 +``` + +## Checking mappings + +There are two ways to check the mappings. + + - Enter the string to get mapped integer + - Enter the mapped integer to get string + +### `UnmapString()` + +The `UnmapString()` function has the full signature `UnmapString(int value, +size_t dimension, size_t unmappingIndex = 0UL)`. + + - `value` is the integer for which you want to find the mapped value + - `dimension` is the dimension in which you want to check the mappings + +```c++ +std::cout << info.UnmapString(0, 2) << "\n"; +std::cout << info.UnmapString(1, 2) << "\n"; +``` + +This will print: + +``` +T +F +``` + +### `UnmapValue()` + +The `UnmapValue()` function has the signature `UnmapValue(const std::string +&input, size_t dimension)`. + + - `input` is the mapped value for which you want to find mapping + - `dimension` is the dimension in which you want to find the mapped value + +```c++ +std::cout << info.UnmapValue("T", 2) << "\n"; +std::cout << info.UnmapValue("F", 2) << "\n"; +``` + +will produce: + +``` +0 +1 +``` + +## Further documentation + +For further documentation on `DatasetMapper` and its uses, see the comments in +the source code in `src/mlpack/core/data/`, as well as its uses in the [examples +repository](https://github.com/mlpack/examples). diff --git a/doc/tutorials/det/det.txt b/doc/tutorials/det.md similarity index 60% rename from doc/tutorials/det/det.txt rename to doc/tutorials/det.md index f819fe87ee..509bdc7ef0 100644 --- a/doc/tutorials/det/det.txt +++ b/doc/tutorials/det.md @@ -1,20 +1,13 @@ -/*! - -@file det.txt -@author Parikshit Ram -@brief Tutorial for how to perform density estimation with Density Estimation Trees (DET). - -@page dettutorial Density Estimation Tree (DET) tutorial - -@section intro_det_tut Introduction +# Density estimation tree (DET) tutorial DETs perform the unsupervised task of density estimation using decision trees. Using a trained density estimation tree (DET), the density at any particular -point can be estimated very quickly (O(log n) time, where n is the number of +point can be estimated very quickly (`O(log n)` time, where `n` is the number of points the tree is built on). The details of this work is presented in the following paper: -@code + +``` @inproceedings{ram2011density, title={Density estimation trees}, author={Ram, P. and Gray, A.G.}, @@ -24,36 +17,27 @@ The details of this work is presented in the following paper: year={2011}, organization={ACM} } -@endcode +``` -\b mlpack provides: +mlpack provides: - - a \ref cli_det_tut "simple command-line executable" to perform density estimation and related analyses using DETs - - a \ref dtree_det_tut "generic C++ class (DTree)" which provides various functionality for the DETs - - a set of functions in the namespace \ref dtutils_det_tut "mlpack::det" to perform cross-validation for the task of density estimation with DETs + - a simple command-line executable to perform density estimation and related + analyses using DETs + - a generic C++ class (`DTree`) which provides various functionality for the + DETs + - a set of functions in the namespace `mlpack::det` to perform cross-validation + for the task of density estimation with DETs -@section toc_det_tut Table of Contents +## Command-line `mlpack_det` -A list of all the sections this tutorial contains. +*(Note: this section was written for the command-line program `mlpack_det`, but +a `det()` function is available for other languages via mlpack's bindings +system. The options are so similar that it is easy to adapt the examples here +to another language.)* - - \ref intro_det_tut - - \ref toc_det_tut - - \ref cli_det_tut - - \ref cli_ex1_de_tut - - \ref cli_ex2_de_test_tut - - \ref cli_ex4_de_vi_tut - - \ref cli_ex6_de_save - - \ref cli_ex7_de_load - - \ref dtree_det_tut - - \ref dtree_pub_func_det_tut - - \ref dtutils_det_tut - - \ref dtutils_util_funcs - - \ref further_doc_det_tut +The command line arguments of this program can be viewed using the `-h` option: -@section cli_det_tut Command-Line mlpack_det -The command line arguments of this program can be viewed using the \c -h option: - -@code +```sh $ mlpack_det -h Density Estimation With Density Estimation Trees @@ -116,103 +100,103 @@ Options: For further information, including relevant papers, citations, and theory, consult the documentation found at http://www.mlpack.org or included with your distribution of mlpack. -@endcode +``` -@subsection cli_ex1_de_tut Plain-vanilla density estimation +### Plain-vanilla density estimation -We can just train a DET on the provided data set \e S. Like all datasets -\b mlpack uses, the data should be row-major (\b mlpack transposes data when it -is loaded; internally, the data is column-major -- see \ref matrices "this page" -for more information). +We can just train a DET on the provided data set `S`. Like all datasets +mlpack uses, the data should be row-major (mlpack transposes data when it +is loaded; internally, the data is column-major---see [this +page](../user/matrices.md) for more information). -@code +```sh $ mlpack_det -t dataset.csv -v -@endcode +``` -By default, \c mlpack_det performs 10-fold cross-validation (using the -\f$\alpha\f$-pruning regularization for decision trees). To perform LOOCV +By default, `mlpack_det` performs 10-fold cross-validation (using the +alpha-pruning regularization for decision trees). To perform LOOCV (leave-one-out cross-validation), which can provide better results but will take longer, use the following command: -@code +```sh $ mlpack_det -t dataset.csv -f 0 -v -@endcode +``` -To perform k-fold crossvalidation, use \c -f \c k (or \c --folds \c k). There -are certain other options available for training. For example, in the -construction of the initial tree, you can specify the maximum and minimum leaf -sizes. By default, they are 10 and 5 respectively; you can set them using the \c --M (\c --max_leaf_size) and the \c -N (\c --min_leaf_size) options. +To perform `k`-fold crossvalidation, use `-f k` (or `--folds k`). There are +certain other options available for training. For example, in the construction +of the initial tree, you can specify the maximum and minimum leaf sizes. By +default, they are 10 and 5 respectively; you can set them using the `-M` +(`--max_leaf_size`) and the `-N` (`--min_leaf_size`) options. -@code +```sh $ mlpack_det -t dataset.csv -M 20 -N 10 -@endcode +``` In case you want to output the density estimates at the points in the training -set, use the \c -e (\c --training_set_estimates_file) option to specify the -output file to which the estimates will be saved. The first line in -density_estimates.txt will correspond to the density at the first point in the +set, use the `-e` (`--training_set_estimates_file`) option to specify the output +file to which the estimates will be saved. The first line in +`density_estimates.txt` will correspond to the density at the first point in the training set. Note that the logarithm of the density estimates are given, which allows smaller estimates to be saved. -@code +```sh $ mlpack_det -t dataset.csv -e density_estimates.txt -v -@endcode +``` -@subsection cli_ex2_de_test_tut Estimation on a test set +### Estimation on a test set Often, it is useful to train a density estimation tree on a training set and then obtain density estimates from the learned estimator for a separate set of -test points. The \c -T (\c --test_file) option allows specification of a set of -test points, and the \c -E (\c --test_set_estimates_file) option allows +test points. The `-T` (`--test_file`) option allows specification of a set of +test points, and the `-E` (`--test_set_estimates_file`) option allows specification of the file into which the test set estimates are saved. Note that the logarithm of the density estimates are saved; this allows smaller values to be saved. -@code +```sh $ mlpack_det -t dataset.csv -T test_points.csv -E test_density_estimates.txt -v -@endcode +``` -@subsection cli_ex4_de_vi_tut Computing the variable importance +### Computing the variable importance The variable importance (with respect to density estimation) of the different -features in the data set can be obtained by using the \c -i (\c --vi_file ) -option. This outputs the absolute (as opposed to relative) variable importance -of the all the features into the specified file. +features in the data set can be obtained by using the `-i` (`--vi_file`) option. +This outputs the absolute (as opposed to relative) variable importance of the +all the features into the specified file. -@code +```sh $ mlpack_det -t dataset.csv -i variable_importance.txt -v -@endcode +``` -@subsection cli_ex6_de_save Saving trained DETs +### Saving trained DETs -The \c mlpack_det program is capable of saving a trained DET to a file for later -usage. The \c --output_model_file or \c -M option allows specification of the -file to save to. In the example below, a DET trained on \c dataset.csv is saved -to the file \c det.xml. +The `mlpack_det` program is capable of saving a trained DET to a file for later +usage. The `--output_model_file` or `-M` option allows specification of the +file to save to. In the example below, a DET trained on `dataset.csv` is saved +to the file `det.xml`. -@code +```sh $ mlpack_det -t dataset.csv -M det.xml -v -@endcode +``` -@subsection cli_ex7_de_load Loading trained DETs +### Loading trained DETs A saved DET can be used to perform any of the functionality in the examples -above. A saved DET is loaded with the \c --input_model_file or \c -m option. -The example below loads a saved DET from \c det.xml and outputs density -estimates on the dataset \c test_dataset.csv into the file \c estimates.csv. +above. A saved DET is loaded with the `--input_model_file` or `-m` option. The +example below loads a saved DET from `det.xml` and outputs density estimates on +the dataset `test_dataset.csv` into the file `estimates.csv`. -@code +```sh $ mlpack_det -m det.xml -T test_dataset.csv -E estimates.csv -v -@endcode +``` -@section dtree_det_tut The 'DTree' class +## The `DTree` class This class implements density estimation trees. Below is a simple example which initializes a density estimation tree. -@code -#include +```c++ +#include using namespace mlpack::det; @@ -222,16 +206,16 @@ extern arma::Mat data; // Initialize the tree. This function also creates and saves the bounding box // of the data. Note that it does not actually build the tree. DTree<> det(data); -@endcode +``` -@subsection dtree_pub_func_det_tut Public Functions +### Public functions -The function \c Grow() greedily grows the tree, adding new points to the tree. +The function `Grow()` greedily grows the tree, adding new points to the tree. Note that the points in the dataset will be reordered. This should only be run on a tree which has not already been built. In general, it is more useful to -use the \c Trainer() function found in \ref dtutils_det_tut. +use the `Trainer()` function, detailed later. -@code +```c++ // This keeps track of the data during the shuffle that occurs while growing the // tree. arma::Col oldFromNew(data.n_cols); @@ -244,24 +228,24 @@ size_t maxLeafSize = 10; size_t minLeafSize = 5; double alpha = det.Grow(data, oldFromNew, false, maxLeafSize, minLeafSize); -@endcode +``` -Note that the alternate volume regularization should not be used (see ticket -#238). +Note that the alternate volume regularization should not be used (see +[#238](https://github.com/mlpack/mlpack/issues/238)). To estimate the density at a given query point, use the following code. Note that the logarithm of the density is returned. -@code +```c++ // For a given query, you can obtain the density estimate. extern arma::Col query; extern DTree* det; double estimate = det->ComputeValue(&query); -@endcode +``` -Computing the \b variable \b importance of each feature for the given DET. +Computing the *variable importance* of each feature for the given DET. -@code +```c++ // The data matrix and density estimation tree. extern arma::mat data; extern DTree* det; @@ -271,23 +255,23 @@ arma::Col varImps; // You can obtain the variable importance from the current tree. det->ComputeVariableImportance(varImps); -@endcode +``` -@section dtutils_det_tut 'namespace mlpack::det' +## The `mlpack::det` namespace The functions in this namespace allows the user to perform tasks with the -'DTree' class. Most importantly, the \c Trainer() method allows the full +`DTree` class. Most importantly, the `Trainer()` method allows the full training of a density estimation tree with cross-validation. There are also utility functions which allow printing of leaf membership and variable importance. -@subsection dtutils_util_funcs Utility Functions +### Utility functions The code below details how to train a density estimation tree with cross-validation. -@code -#include +```c++ +#include using namespace mlpack::det; @@ -302,15 +286,15 @@ const size_t minLeafSize = 5; // Train the density estimation tree with cross-validation. DTree<>* dtree_opt = Trainer(data, folds, false, maxLeafSize, minLeafSize); -@endcode +``` Note that the alternate volume regularization should be set to false because it -has known bugs (see #238). +has known bugs (see [#238](https://github.com/mlpack/mlpack/issues/238)).. To print the class membership of leaves in the tree into a file, see the following code. -@code +```c++ extern arma::Mat labels; extern DTree* det; const size_t numClasses = 3; // The number of classes must be known. @@ -318,57 +302,21 @@ const size_t numClasses = 3; // The number of classes must be known. extern string leafClassMembershipFile; PrintLeafMembership(det, data, labels, numClasses, leafClassMembershipFile); -@endcode +``` -Note that you can find the number of classes with \c max(labels) \c + \c 1. -The variable importance can also be printed to a file in a similar manner. +Note that you can find the number of classes with `max(labels) + 1`. The +variable importance can also be printed to a file in a similar manner. -@code +```c++ extern DTree* det; extern string variableImportanceFile; const size_t numFeatures = data.n_rows; PrintVariableImportance(det, numFeatures, variableImportanceFile); -@endcode +``` -@section further_doc_det_tut Further Documentation -For further documentation on the DTree class, consult the -\ref mlpack::det::DTree "complete API documentation". +## Further documentation -*/ - ------ this option is not available in DET right now; see #238! ----- -@subsection cli_alt_reg_tut Alternate DET regularization - -The usual regularized error \f$R_\alpha(t)\f$ of a node \f$t\f$ is given by: -\f$R_\alpha(t) = R(t) + \alpha |\tilde{t}|\f$ where - -\f{ -R(t) = -\frac{|t|^2}{N^2 V(t)}. -\f} - -\f$V(t)\f$ is the volume of the node \f$t\f$ and \f$\tilde{t}\f$ is -the set of leaves in the subtree rooted at \f$t\f$. - -For the purposes of density estimation, there is a different form of -regularization: instead of penalizing the number of leaves in the subtree, we -penalize the sum of the inverse of the volumes of the leaves. With this -regularization, very small volume nodes are discouraged unless the data actually -warrants it. Thus, - -\f[ -R_\alpha'(t) = R(t) + \alpha I_v(\tilde{t}) -\f] - -where - -\f[ -I_v(\tilde{t}) = \sum_{l \in \tilde{t}} \frac{1}{V(l)}. -\f] - -To use this form of regularization, use the \c -R flag. - -@code -$ mlpack_det -t dataset.csv -R -v -@endcode +For further documentation on the `DTree` class, consult the comments in the +source code, in `mlpack/methods/det/`. diff --git a/doc/tutorials/emst.md b/doc/tutorials/emst.md new file mode 100644 index 0000000000..115e91f9df --- /dev/null +++ b/doc/tutorials/emst.md @@ -0,0 +1,136 @@ +# EMST Tutorial + +The Euclidean Minimum Spanning Tree problem is widely used in machine learning +and data mining applications. Given a set `S` of points in `R^d`, our task is +to compute lowest weight spanning tree in the complete graph on `S` with edge +weights given by the Euclidean distance between points. + +Among other applications, the EMST can be used to compute hierarchical +clusterings of data. A *single-linkage clustering* can be obtained from the +EMST by deleting all edges longer than a given cluster length. This technique +is also referred to as a *Friends-of-Friends* clustering in the astronomy +literature. + +mlpack includes an implementation of ***Dual-Tree Boruvka*** which uses +`kd`-trees by default; this is the empirically and theoretically fastest EMST +algorithm. In addition, the implementation supports the use of different trees +via templates. For more details, see the following paper: + +``` +@inproceedings{march2010fast, + title={Fast {E}uclidean minimum spanning tree: algorithm, analysis, and +applications}, + author={March, William B. and Ram, Parikshit and Gray, Alexander G.}, + booktitle={Proceedings of the 16th ACM SIGKDD International Conference on +Knowledge Discovery and Data Mining (KDD '10)}, + pages={603--612}, + year={2010}, + organization={ACM} +} +``` + +mlpack provides: + + - a simple command-line executable to compute the EMST of a given data set + - a simple C++ interface to compute the EMST + +## Command-line `mlpack_emst` + +The `mlpack_emst` program in mlpack will compute the EMST of a given set +of points and store the resulting edge list to a file. Note that mlpack also +has bindings to other languages, and so there also exists, e.g., an `emst()` +function in Python and other similar functions in other languages. Although +these examples are written for the command-line `mlpack_emst` program, it is +easy to adapt each of these to another language. + +The output file contains an edge list representation of the MST in an `(n - 1) x +3` matrix, where the first and second columns are labels of points and the third +column is the edge weight. The edges are sorted in order of increasing weight. + +Below are several examples of simple usage (and the resultant output). The `-v` +option is used so that verbose output is given. Further documentation on each +individual option can be found by typing + +```sh +$ mlpack_emst --help +``` + +```sh +$ mlpack_emst --input_file=dataset.csv --output_file=edge_list.csv -v +[INFO ] Reading in data. +[INFO ] Loading 'dataset.csv' as CSV data. +[INFO ] Data read, building tree. +[INFO ] Tree built, running algorithm. +[INFO ] 4 edges found so far. +[INFO ] 5 edges found so far. +[INFO ] Total spanning tree length: 1002.45 +[INFO ] Saving CSV data to 'edge_list.csv'. +[INFO ] +[INFO ] Execution parameters: +[INFO ] help: false +[INFO ] info: "" +[INFO ] input_file: dataset.csv +[INFO ] leaf_size: 1 +[INFO ] naive: false +[INFO ] output_file: edge_list.csv +[INFO ] verbose: true +[INFO ] +[INFO ] Program timers: +[INFO ] emst/mst_computation: 0.000179s +[INFO ] emst/tree_building: 0.000061s +[INFO ] total_time: 0.052641s +``` + +The code performs at most `log N` iterations for `N` data points. It will print +an update on the number of MST edges found after each iteration. Convenient +program timers are given for different parts of the calculation at the bottom of +the output, as well as the parameters the simulation was run with. + +```sh +$ cat dataset.csv +0, 0 +1, 1 +3, 3 +0.5, 0 +1000, 0 +1001, 0 + +$ cat edge_list.csv +0.0000000000e+00,3.0000000000e+00,5.0000000000e-01 +4.0000000000e+00,5.0000000000e+00,1.0000000000e+00 +1.0000000000e+00,3.0000000000e+00,1.1180339887e+00 +1.0000000000e+00,2.0000000000e+00,2.8284271247e+00 +2.0000000000e+00,4.0000000000e+00,9.9700451353e+02 +``` + +The input points are labeled 0-5. The output tells us that the MST connects +point 0 to point 3, point 4 to point 5, point 1 to point 3, point 1 to point 2, +and point 2 to point 4, with the corresponding edge weights given in the third +column. The total length of the MST is also given in the verbose output. + +Note that it is also possible to compute the EMST using a naive (`O(N^2)`) +algorithm for timing and comparison purposes, using the `--naive` option. + +## The `DualTreeBoruvka` class + +The `DualTreeBoruvka` class contains our implementation of the Dual-Tree Boruvka +algorithm. + +The class has two constructors: the first takes the data set, constructs the +tree (where the type of tree constructed is the TreeType template parameter), +and computes the MST. The second takes data set and an already constructed +tree. + +The class provides one method that performs the MST computation: + +```c++ +void ComputeMST(const arma::mat& results); +``` + +This method stores the computed MST in the matrix results in the format given +above. + +## Further documentation + +For further documentation on the `DualTreeBoruvka` class, consult the comments +in the source code, in `mlpack/methods/emst/dtb.hpp`. diff --git a/doc/tutorials/emst/emst.txt b/doc/tutorials/emst/emst.txt deleted file mode 100644 index bfc17bb9c4..0000000000 --- a/doc/tutorials/emst/emst.txt +++ /dev/null @@ -1,148 +0,0 @@ -/*! - -@file emst.txt -@author Bill March -@brief Tutorial for the Euclidean Minimum Spanning Tree algorithm. - -@page emst_tutorial EMST Tutorial - -@section intro_emsttut Introduction - -The Euclidean Minimum Spanning Tree problem is widely used in machine learning -and data mining applications. Given a set \f$S\f$ of points in \f$\mathbf{R}^d\f$, -our task is to compute lowest weight spanning tree in the complete graph on \f$S\f$ -with edge weights given by the Euclidean distance between points. - -Among other applications, the EMST can be used to compute hierarchical clusterings -of data. A single-linkage clustering can be obtained from the EMST by deleting -all edges longer than a given cluster length. This technique is also referred to as a Friends-of-Friends clustering in the astronomy literature. - -mlpack includes an implementation of Dual-Tree Boruvka which uses -\f$kd\f$-trees by default; this is the empirically and theoretically fastest -EMST algorithm. In addition, the implementation supports the use of different -trees via templates. For more details, see the following paper: - -@code -@inproceedings{march2010fast, - title={Fast {E}uclidean minimum spanning tree: algorithm, analysis, and -applications}, - author={March, William B. and Ram, Parikshit and Gray, Alexander G.}, - booktitle={Proceedings of the 16th ACM SIGKDD International Conference on -Knowledge Discovery and Data Mining (KDD '10)}, - pages={603--612}, - year={2010}, - organization={ACM} -} -@endcode - -\b mlpack provides: - - - a \ref cli_emsttut "simple command-line executable" to compute the EMST of a given data set - - a \ref dtb_emsttut "simple C++ interface" to compute the EMST - -@section toc_emsttut Table of Contents - -A list of all the sections this tutorial contains. - - - \ref intro_emsttut - - \ref toc_emsttut - - \ref cli_emsttut - - \ref dtb_emsttut - - \ref further_doc_emsttut - -@section cli_emsttut Command-Line 'EMST' - -The \c mlpack_emst executable in \b mlpack will compute the EMST of a given set -of points and store the resulting edge list to a file. - -The output file contains an edge list representation of the MST in an -\f$n-1 \times 3 \f$ matrix, where the first and second columns are labels of -points and the third column is the edge weight. The edges are sorted in order -of increasing weight. - -Below are several examples of simple usage (and the resultant output). The -\c -v option is used so that verbose output is given. Further documentation on -each individual option can be found by typing - -@code -$ mlpack_emst --help -@endcode - -@code -$ mlpack_emst --input_file=dataset.csv --output_file=edge_list.csv -v -[INFO ] Reading in data. -[INFO ] Loading 'dataset.csv' as CSV data. -[INFO ] Data read, building tree. -[INFO ] Tree built, running algorithm. -[INFO ] 4 edges found so far. -[INFO ] 5 edges found so far. -[INFO ] Total spanning tree length: 1002.45 -[INFO ] Saving CSV data to 'edge_list.csv'. -[INFO ] -[INFO ] Execution parameters: -[INFO ] help: false -[INFO ] info: "" -[INFO ] input_file: dataset.csv -[INFO ] leaf_size: 1 -[INFO ] naive: false -[INFO ] output_file: edge_list.csv -[INFO ] verbose: true -[INFO ] -[INFO ] Program timers: -[INFO ] emst/mst_computation: 0.000179s -[INFO ] emst/tree_building: 0.000061s -[INFO ] total_time: 0.052641s -@endcode - -The code performs at most \f$\log N\f$ iterations for \f$N\f$ data points. It will print an update on the number of MST edges found after each iteration. -Convenient program timers are given for different parts of the calculation at -the bottom of the output, as well as the parameters the simulation was run with. - -@code -$ cat dataset.csv -0, 0 -1, 1 -3, 3 -0.5, 0 -1000, 0 -1001, 0 - -$ cat edge_list.csv -0.0000000000e+00,3.0000000000e+00,5.0000000000e-01 -4.0000000000e+00,5.0000000000e+00,1.0000000000e+00 -1.0000000000e+00,3.0000000000e+00,1.1180339887e+00 -1.0000000000e+00,2.0000000000e+00,2.8284271247e+00 -2.0000000000e+00,4.0000000000e+00,9.9700451353e+02 -@endcode - -The input points are labeled 0-5. The output tells us that the MST connects -point 0 to point 3, point 4 to point 5, point 1 to point 3, point 1 to point 2, -and point 2 to point 4, with the corresponding edge weights given in the third -column. The total length of the MST is also given in the verbose output. - -Note that it is also possible to compute the EMST using a naive (\f$O(N^2)\f$) -algorithm for timing and comparison purposes, using the \c --naive option. - -@section dtb_emsttut The 'DualTreeBoruvka' class - -The 'DualTreeBoruvka' class contains our implementation of the Dual-Tree Boruvka -algorithm. - -The class has two constructors: the first takes the data set, constructs the -tree (where the type of tree constructed is the TreeType template parameter), -and computes the MST. The second takes data set and an already constructed -tree. - -The class provides one method that performs the MST computation: -@code -void ComputeMST(const arma::mat& results); -@endcode - -This method stores the computed MST in the matrix results in the format given above. - -@section further_doc_emsttut Further documentation - -For further documentation on the DualTreeBoruvka class, consult the -\ref mlpack::emst::DualTreeBoruvka "complete API documentation". - -*/ diff --git a/doc/tutorials/fastmks.md b/doc/tutorials/fastmks.md new file mode 100644 index 0000000000..8dacb85553 --- /dev/null +++ b/doc/tutorials/fastmks.md @@ -0,0 +1,554 @@ +# Fast max-kernel search tutorial (FastMKS) + +The FastMKS algorithm (fast exact max-kernel search) is a recent algorithm +proposed in the following papers: + +``` +@inproceedings{curtin2013fast, + title={Fast Exact Max-Kernel Search}, + author={Curtin, Ryan R. and Ram, Parikshit and Gray, Alexander G.}, + booktitle={Proceedings of the 2013 SIAM International Conference on Data + Mining (SDM '13)}, + year={2013}, + pages={1--9} +} + +@article{curtin2014dual, + author = {Curtin, Ryan R. and Ram, Parikshit}, + title = {Dual-tree fast exact max-kernel search}, + journal = {Statistical Analysis and Data Mining}, + volume = {7}, + number = {4}, + publisher = {Wiley Subscription Services, Inc., A Wiley Company}, + issn = {1932-1872}, + url = {http://dx.doi.org/10.1002/sam.11218}, + doi = {10.1002/sam.11218}, + pages = {229--253}, + year = {2014}, +} +``` + +Given a set of query points `Q` and a set of reference points `R`, the FastMKS +algorithm is a fast dual-tree (or single-tree) algorithm which finds + +``` +argmax_{p_r in R} K(p_q, p_r) +``` + +for all points `p_q` in `Q` and for some Mercer kernel `K()`. A Mercer kernel +is a kernel that is positive semidefinite; these are the classes of kernels that +can be used with the kernel trick. In short, the positive semidefiniteness of a +Mercer kernel means that any kernel matrix (or Gram matrix) created on a dataset +must be positive semidefinite. + +The FastMKS algorithm builds trees on the datasets `Q` and `R` in such a way +that explicit representation of the points in the kernel space is unnecessary, +by using cover trees (see `mlpack::tree::CoverTree`). This allows the algorithm +to be run, for instance, on string kernels, where there is no sensible explicit +representation. The mlpack implementation allows any type of tree that does not +require an explicit representation to be used. For more details, see the paper. + +At the time of this writing there is no other fast algorithm for exact +max-kernel search. mlpack implements both single-tree and dual-tree fast +max-kernel search. + +mlpack provides: + + - a simple command-line executable to run FastMKS + - a C++ interface to run FastMKS + +## Command-line FastMKS (`mlpack_fastmks`) + +mlpack provides a command-line program, `mlpack_fastmks`, which is used to +perform FastMKS on a given query and reference dataset. It supports numerous +different types of kernels: + + - `mlpack::kernel::LinearKernel` + - `mlpack::kernel::PolynomialKernel` + - `mlpack::kernel::CosineDistance` + - `mlpack::kernel::GaussianKernel` + - `mlpack::kernel::EpanechnikovKernel` + - `mlpack::kernel::TriangularKernel` + - `mlpack::kernel::HyperbolicTangentKernel` + +Note that when a shift-invariant kernel is used, the results will be the same as +nearest neighbor search, so [KNN](neighbor_search.md) may be a better option. A +shift-invariant kernel is a kernel that depends only on the distance between the +two input points. The `mlpack::kernel::GaussianKernel`, +`mlpack::kernel::EpanechnikovKernel`, and `mlpack::kernel::TriangularKernel` are +instances of shift-invariant kernels. The paper contains more details on this +situation. The `mlpack_fastmks` executable still provides these kernels as +options, though. + +The following examples detail usage of the `mlpack_fastmks` program. Note that +you can get documentation on all the possible parameters by typing: + +```sh +$ mlpack_fastmks --help +``` + +### FastMKS with a linear kernel on one dataset + +If only one dataset is specified (with `-r` or `--reference_file`), the +reference dataset is taken to be both the query and reference datasets. The +example below finds the 4 maximum kernels of each point in `dataset.csv`, using +the default linear kernel. + +```sh +$ mlpack_fastmks -r dataset.csv -k 4 -v -p products.csv -i indices.csv +``` + +When the operation completes, the values of the kernels are saved in +`products.csv` and the indices of the points which give the maximum kernels are +saved in `indices.csv`. + +```sh +$ head indices.csv +762,910,863,890 +762,910,426,568 +910,762,863,426 +762,910,863,426 +863,910,614,762 +762,863,910,614 +762,910,488,568 +762,910,863,426 +910,762,863,426 +863,762,910,614 +``` + +```sh +$ head products.csv +1.6221652894e+00,1.5998743443e+00,1.5898890769e+00,1.5406789753e+00 +1.3387953449e+00,1.3317349486e+00,1.2966613184e+00,1.2774493620e+00 +1.6386110476e+00,1.6332029753e+00,1.5952629124e+00,1.5887195330e+00 +1.0917545803e+00,1.0820878726e+00,1.0668992636e+00,1.0419838050e+00 +1.2272441028e+00,1.2169643942e+00,1.2104597963e+00,1.2067780154e+00 +1.5720962456e+00,1.5618504956e+00,1.5609069923e+00,1.5235605095e+00 +1.3655478674e+00,1.3548593212e+00,1.3311547298e+00,1.3250728881e+00 +2.0119149744e+00,2.0043668067e+00,1.9847289214e+00,1.9298280046e+00 +1.1586923205e+00,1.1494586097e+00,1.1274872962e+00,1.1248172766e+00 +4.4789820372e-01,4.4618539778e-01,4.4200024852e-01,4.3989721792e-01 +``` + +We can see in this example that for point 0, the point with maximum kernel value +is point 762, with a kernel value of 1.622165. For point 3, the point with +third largest kernel value is point 863, with a kernel value of 1.0669. + +### FastMKS on a reference and query dataset + +The query points may be different than the reference points. To specify a +different query set, the `-q` (or `--query_file`) option is used, as in the +example below. + +```sh +$ mlpack_fastmks -q query_set.csv -r reference_set.csv -k 5 -i indices.csv \ +> -p products.csv +``` + +### FastMKS with a different kernel + +The `mlpack_fastmks` program offers more than just the linear kernel. Valid +options are `'linear'`, `'polynomial'`, `'cosine'`, `'gaussian'`, +`'epanechnikov'`, `'triangular'` and `'hyptan'` (the hyperbolic tangent kernel). +Note that the hyperbolic tangent kernel is provably not a Mercer kernel but is +positive semidefinite on most datasets and is commonly used as a kernel. Note +also that the Gaussian kernel and other shift-invariant kernels give the same +results as nearest neighbor search (see [the tutorial](neighbor_search.md)). + +The kernel to use is specified with the `-K` (or `--kernel`) option. The +example below uses the cosine similarity as a kernel. + +```sh +$ mlpack_fastmks -r dataset.csv -k 5 -K cosine -i indices.csv -p products.csv -v +``` + +### Using single-tree search or naive search + +In some cases, it may be useful to not use the dual-tree FastMKS algorithm. +Instead you can specify the `--single` option, indicating that a tree should be +built only on the reference set, and then the queries should be processed in a +linear scan (instead of in a tree). Alternately, the `-N` (or `--naive`) option +makes the program not build trees at all and instead use brute-force search to +find the solutions. + +The example below uses single-tree search on two datasets with the linear +kernel. + +```sh +$ mlpack_fastmks -q query_set.csv -r reference_set.csv --single -k 5 \ +> -p products.csv -i indices.csv -K linear +``` + +The example below uses naive search on one dataset. + +```sh +$ mlpack_fastmks -r reference_set.csv -k 5 -N -p products.csv -i indices.csv +``` + +### Parameters for alternate kernels + +Many of the alternate kernel choices have parameters which can be chosen; these +are detailed in this section. + + - `-w` (`--bandwidth`): this sets the bandwidth of the kernel, and is + applicable to the `'gaussian'`, `'epanechnikov'`, and `'triangular'` kernels. + This is the "spread" of the kernel. + + - `-d` (`--degree`): this sets the degree of the polynomial kernel (the power + to which the result is raised). It is only applicable to the `'polynomial'` + kernel. + + - `-o` (`--offset`): this sets the offset of the kernel, for the + `'polynomial'` and `'hyptan'` kernel. See the documentation for + `mlpack::kernel::PolynomialKernel` and + `mlpack::kernel::HyperbolicTangentKernel` for more information. + + - `-s` (`--scale`): this sets the scale of the kernel, and is only applicable + to the `'hyptan'` kernel. See the documentation for + `mlpack::kernel::HyperbolicTangentKernel` for more information. + +### Saving a FastMKS model/tree + +The `mlpack_fastmks` program also supports saving a model built on a reference +dataset (this model includes the tree, the kernel, and the search parameters). +The `--output_model_file` or `-M` option allows one to save these parameters to +disk for later usage. An example is below: + +```sh +$ mlpack_fastmks -r reference_set.csv -K cosine -M fastmks_model.xml +``` + +This example builds a tree on the dataset in `reference_set.csv` using the +cosine similarity kernel, and saves the resulting model to `fastmks_model.xml`. +This model may then be used in later calls to the `mlpack_fastmks` program. + +### Loading a FastMKS model for further searches + +Supposing that a FastMKS model has been saved with the `--output_model_file` or +`-M` parameter, that model can then be later loaded in subsequent calls to the +`mlpack_fastmks` program, using the `--input_model_file` or `-m` option. For +instance, with a model saved in `fastmks_model.xml` and a query set in +`query_set.csv`, we can find 3 max-kernel candidates, saving to `indices.csv` +and `kernels.csv`: + +```sh +$ mlpack_fastmks -m fastmks_model.xml -k 3 -i indices.csv -p kernels.csv +``` + +Loading a model as opposed to building a model is advantageous because the +reference tree is already built. So, among other situations, this could be +useful in the setting where many different query sets (or many different values +of `k`) will be used. + +Note that the kernel cannot be changed in a saved model without rebuilding the +model entirely. + +## The `FastMKS` class + +The `FastMKS<>` class offers a simple API for use within C++ applications, and +allows further flexibility in kernel choice and tree type choice. However, +`FastMKS<>` has no default template parameter for the kernel type---that must be +manually specified. Choices that mlpack provides include: + + - `mlpack::kernel::LinearKernel` + - `mlpack::kernel::PolynomialKernel` + - `mlpack::kernel::CosineDistance` + - `mlpack::kernel::GaussianKernel` + - `mlpack::kernel::EpanechnikovKernel` + - `mlpack::kernel::TriangularKernel` + - `mlpack::kernel::HyperbolicTangentKernel` + - `mlpack::kernel::LaplacianKernel` + - `mlpack::kernel::PSpectrumStringKernel` + +The following examples use kernels from that list. Writing your own kernel is +detailed in the next section. Remember that when you are using the C++ +interface, the data matrices must be column-major. See the [matrices +documentation](../user/matrices.md) for more information. + +### `FastMKS` on one dataset + +Given only a reference dataset, the following code will run FastMKS with k set +to 5. + +```c++ +#include + +using namespace mlpack::fastmks; + +// The reference dataset, which is column-major. +extern arma::mat data; + +// This will initialize the FastMKS object with the linear kernel with default +// options: K(x, y) = x^T y. The tree is built in the constructor. +FastMKS f(data); + +// The results will be stored in these matrices. +arma::Mat indices; +arma::mat products; + +// Run FastMKS. +f.Search(5, indices, products); +``` + +### FastMKS with a query and reference dataset + +In this setting we have both a query and reference dataset. We search for 10 +maximum kernels. + +``` +#include + +using namespace mlpack::fastmks; +using namespace mlpack::kernel; + +// The reference and query datasets, which are column-major. +extern arma::mat referenceData; +extern arma::mat queryData; + +// This will initialize the FastMKS object with the triangular kernel with +// default options (bandwidth of 1). The reference tree is built in the +// constructor. +FastMKS f(referenceData); + +// The results will be stored in these matrices. +arma::Mat indices; +arma::mat products; + +// Run FastMKS. The query tree is built during the call to Search(). +f.Search(queryData, 10, indices, products); +``` + +### FastMKS with an initialized kernel + +Often, kernels have parameters which need to be specified. `FastMKS<>` has +constructors which take initialized kernels. Note that temporary kernels cannot +be passed as an argument. The example below initializes a `PolynomialKernel` +object and then runs FastMKS with a query and reference dataset. + +```c++ +#include + +using namespace mlpack::fastmks; +using namespace mlpack::kernel; + +// The reference and query datasets, which are column-major. +extern arma::mat referenceData; +extern arma::mat queryData; + +// Initialize the polynomial kernel with degree of 3 and offset of 2.5. +PolynomialKernel pk(3.0, 2.5); + +// Create the FastMKS object with the initialized kernel. +FastMKS f(referenceData, pk); + +// The results will be stored in these matrices. +arma::Mat indices; +arma::mat products; + +// Run FastMKS. +f.Search(queryData, 10, indices, products); +``` + +The syntax for running FastMKS with one dataset and an initialized kernel is +very similar: + +```c++ +f.Search(10, indices, products); +``` + +### FastMKS with an already-created tree + +By default, `FastMKS<>` uses the cover tree datastructure (see the +`mlpack::tree::CoverTree` documentation). Sometimes, it is useful to modify the +parameters of the cover tree. In this scenario, a tree must be built outside of +the constructor, and then passed to the appropriate `FastMKS<>` constructor. An +example on just a reference dataset is shown below, where the base of the cover +tree is modified. + +We also use an instantiated kernel, but because we are building our own tree, we +must use `mlpack::metric::IPMetric` so that our tree is built on the metric +induced by our kernel function. + +```c++ +#include + +// The reference dataset, which is column-major. +extern arma::mat data; + +// Initialize the polynomial kernel with a degree of 4 and offset of 2.0. +PolynomialKernel pk(4.0, 2.0); + +// Create the metric induced by this kernel (because a kernel is not a metric +// and we can't build a tree on a kernel alone). +IPMetric metric(pk); + +// Now build a tree on the reference dataset using the instantiated metric and +// the custom base of 1.5 (default is 1.3). We have to be sure to use the right +// type here -- FastMKS needs the FastMKSStat object as the tree's +// StatisticType. +typedef tree::CoverTree, tree::FirstPointIsRoot, + FastMKSStat> TreeType; // Convenience typedef. +TreeType* tree = new TreeType(data, metric, 1.5); + +// Now initialize FastMKS with that statistic. We don't need to specify the +// TreeType template parameter since we are still using the default. We don't +// need to pass the kernel because that is contained in the tree. +FastMKS f(tree); + +// The results will be stored in these matrices. +arma::Mat indices; +arma::mat products; + +// Run FastMKS. +f.Search(10, indices, products); +``` + +The syntax is similar for the case where different query and reference datasets +are given; but trees for both need to be built in the manner specified above. +Be sure to build both trees using the same metric (or at least a metric with the +exact same parameters). + +```c++ +f.Search(queryTree, 10, indices, products); +``` + +### Writing a custom kernel for FastMKS + +While mlpack provides some number of kernels in the `mlpack::kernel` namespace, +it is easy to create a custom kernel. To satisfy the [KernelType +policy](../developer/kernels.md), a class must implement the following methods: + +```c++ +// Empty constructor is required. +KernelType(); + +// Evaluate the kernel between two points. +template +double Evaluate(const VecType& a, const VecType& b); +``` + +The template parameter `VecType` is helpful (but not necessary) so that the +kernel can be used with both sparse and dense matrices (`arma::sp_mat` and +`arma::mat`). + +### Using other tree types for FastMKS + +The use of the cover tree is not necessary for FastMKS, although it is the +default tree type. A different type of tree can be specified with the TreeType +template parameter. However, the tree type is required to have +`mlpack::fastmks::FastMKSStat` as the `StatisticType`, and for FastMKS to work, +the tree must be built only on kernel evaluations (or distance evaluations in +the kernel space via `IPMetric::Evaluate()`). + +Below is an example where a custom tree class, `CustomTree`, is used as the +tree type for FastMKS. In this example FastMKS is only run on one dataset. + +```c++ +#include +#include "custom_tree.hpp" + +using namespace mlpack::fastmks; +using namespace mlpack::tree; + +// The dataset that FastMKS will be run on. +extern arma::mat data; + +// The custom tree type. We'll assume that the first template parameter is the +// statistic type. +typedef CustomTree TreeType; + +// The FastMKS constructor will create the tree. +FastMKS f(data); + +// These will hold the results. +arma::Mat indices; +arma::mat products; + +// Run FastMKS. +f.Search(5, indices, products); +``` + +### Running FastMKS on objects + +FastMKS has a lot of utility on objects which are not representable in some sort +of metric space. These objects might be strings, graphs, models, or other +objects. For these types of objects, questions based on distance don't really +make sense. One good example is with strings. The question "how far is 'dog' +from 'Taki Inoue'?" simply doesn't make sense. We can't have a centroid of the +terms 'Fritz', 'E28', and 'popsicle'. + +However, what we can do is define some sort of kernel on these objects. These +kernels generally correspond to some similarity measure, with one example being +the p-spectrum string kernel (see `mlpack::kernel::PSpectrumStringKernel`). +Using that, we can say "how similar is 'dog' to 'Taki Inoue'?" and get an actual +numerical result by evaluating `K('dog', 'Taki Inoue')` (where `K` is our +p-spectrum string kernel). + +The only requirement on these kernels is that they are positive definite kernels +(or Mercer kernels). For more information on those details, refer to the +FastMKS paper. + +Remember that FastMKS is a tree-based method. But trees like the binary space +tree require centroids---and as we said earlier, centroids often don't make +sense with these types of objects. Therefore, we need a type of tree which is +built *exclusively* on points in the dataset---those are points which we can +evaluate our kernel function on. The cover tree is one example of a type of +tree satisfying this condition; its construction will only call the kernel +function on two points that are in the dataset. + +But, we have one more problem. The `CoverTree` class is built on `arma::mat` +objects (dense matrices). Our objects, however, are not necessarily +representable in a column of a matrix. To use the example we have been using, +strings cannot be represented easily in a matrix because they may all have +different lengths. + +The way to work around this problem is to create a "fake" data matrix which +simply holds indices to objects. A good example of how to do this is detailed +in the documentation for the `mlpack::kernel::PSpectrumStringKernel` class. + +In short, the trick is to make each data matrix one-dimensional and containing +linear indices: + +```c++ +arma::mat data = "0 1 2 3 4 5 6 7 8"; +``` + +Then, when `Evaluate()` is called on the kernel function, the parameters will be +two one-dimensional vectors that simply contain indices to objects. The example +below details the process a little better: + +```c++ +// This function evaluates the kernel on two Objects (in this example, its +// implementation is not important; the only important thing is that the +// function exists). +double ObjectKernel::Evaluate(const Object& a, const Object& b) const; + +template +double ObjectKernel::Evaluate(const VecType& a, const VecType& b) const +{ + // Extract the indices from the vectors. + const size_t indexA = size_t(a[0]); + const size_t indexB = size_t(b[0]); + + // Assume that 'objects' is an array (or std::vector or other container) + // holding Objects. + const Object& objectA = objects[indexA]; + const Object& objectB = objects[indexB]; + + // Now call the function that does the actual evaluation on the objects and + // return its result. + return Evaluate(objectA, objectB); +} +``` + +As written earlier, the documentation for +`mlpack::kernel::PSpectrumStringKernel` is a good place to consult for further +reference on this. That kernel uses two dimensional indices; one dimension +represents the index of the string, and the other represents whether it is +referring to the query set or the reference set. If your kernel is meant to +work on separate query and reference sets, that strategy should be considered. + +## Further documentation + +For further documentation on the FastMKS class, consult the documentation in the +source code for FastMKS, in `mlpack/methods/fastmks/`. diff --git a/doc/tutorials/fastmks/fastmks.txt b/doc/tutorials/fastmks/fastmks.txt deleted file mode 100644 index f1fc22835e..0000000000 --- a/doc/tutorials/fastmks/fastmks.txt +++ /dev/null @@ -1,599 +0,0 @@ -/*! - -@file fastmks.txt -@author Ryan Curtin -@brief Tutorial for how to use FastMKS in mlpack. - -@page fmkstutorial Fast max-kernel search tutorial (fastmks) - -@section intro_fmkstut Introduction - -The FastMKS algorithm (fast exact max-kernel search) is a recent algorithm -proposed in the following papers: - -@code -@inproceedings{curtin2013fast, - title={Fast Exact Max-Kernel Search}, - author={Curtin, Ryan R. and Ram, Parikshit and Gray, Alexander G.}, - booktitle={Proceedings of the 2013 SIAM International Conference on Data - Mining (SDM '13)}, - year={2013}, - pages={1--9} -} - -@article{curtin2014dual, - author = {Curtin, Ryan R. and Ram, Parikshit}, - title = {Dual-tree fast exact max-kernel search}, - journal = {Statistical Analysis and Data Mining}, - volume = {7}, - number = {4}, - publisher = {Wiley Subscription Services, Inc., A Wiley Company}, - issn = {1932-1872}, - url = {http://dx.doi.org/10.1002/sam.11218}, - doi = {10.1002/sam.11218}, - pages = {229--253}, - year = {2014}, -} -@endcode - -Given a set of query points \f$Q\f$ and a set of reference points \f$R\f$, the -FastMKS algorithm is a fast dual-tree (or single-tree) algorithm which finds - -\f[ -\arg\max_{p_r \in R} K(p_q, p_r) -\f] - -for all points \f$p_q \in Q\f$ and for some Mercer kernel \f$K(\cdot, \cdot)\f$. -A Mercer kernel is a kernel that is positive semidefinite; these are the classes -of kernels that can be used with the kernel trick. In short, the positive -semidefiniteness of a Mercer kernel means that any kernel matrix (or Gram -matrix) created on a dataset must be positive semidefinite. - -The FastMKS algorithm builds trees on the datasets \f$Q\f$ and \f$R\f$ in such a -way that explicit representation of the points in the kernel space is -unnecessary, by using cover trees (\ref mlpack::tree::CoverTree). This allows -the algorithm to be run, for instance, on string kernels, where there is no -sensible explicit representation. The \b mlpack implementation allows any type -of tree that does not require an explicit representation to be used. For more -details, see the paper. - -At the time of this writing there is no other fast algorithm for exact -max-kernel search. \b mlpack implements both single-tree and dual-tree fast -max-kernel search. - -\b mlpack provides: - - - a \ref cli_fmkstut "simple command-line executable" to run FastMKS - - a \ref fastmks_fmkstut "C++ interface" to run FastMKS - -@section toc_fmkstut Table of Contents - -A list of all the sections this tutorial contains. - - - \ref intro_fmkstut - - \ref toc_fmkstut - - \ref cli_fmkstut - - \ref cli_ex1_fmkstut - - \ref cli_ex2_fmkstut - - \ref cli_ex3_fmkstut - - \ref cli_ex4_fmkstut - - \ref cli_ex5_fmkstut - - \ref cli_ex6_fmkstut - - \ref cli_ex7_fmkstut - - \ref fastmks_fmkstut - - \ref fastmks_ex1_fmkstut - - \ref fastmks_ex2_fmkstut - - \ref fastmks_ex3_fmkstut - - \ref fastmks_ex4_fmkstut - - \ref writing_kernel_fmkstut - - \ref custom_tree_fmkstut - - \ref objects_fmkstut - - \ref further_doc_fmkstut - -@section cli_fmkstut Command-line FastMKS (mlpack_fastmks) - -\b mlpack provides a command-line program, \c mlpack_fastmks, which is used to -perform FastMKS on a given query and reference dataset. It supports numerous -different types of kernels: - - - \ref mlpack::kernel::LinearKernel "linear kernel" - - \ref mlpack::kernel::PolynomialKernel "polynomial kernel" - - \ref mlpack::kernel::CosineDistance "cosine distance" - - \ref mlpack::kernel::GaussianKernel "Gaussian kernel" - - \ref mlpack::kernel::EpanechnikovKernel "Epanechnikov kernel" - - \ref mlpack::kernel::TriangularKernel "triangular kernel" - - \ref mlpack::kernel::HyperbolicTangentKernel "hyperbolic tangent kernel" - -Note that when a shift-invariant kernel is used, the results will be the same as -nearest neighbor search, so @ref nstutorial "KNN" may be a better option. A -shift-invariant kernel is a kernel that depends only on the distance between the -two input points. The \ref mlpack::kernel::GaussianKernel "Gaussian kernel", -\ref mlpack::kernel::EpanechnikovKernel "Epanechnikov kernel", and \ref -mlpack::kernel::TriangularKernel "triangular kernel" are instances of -shift-invariant kernels. The paper contains more details on this situation. -The \c mlpack_fastmks executable still provides these kernels as options, -though. - -The following examples detail usage of the \c mlpack_fastmks program. Note that -you can get documentation on all the possible parameters by typing: - -@code -$ mlpack_fastmks --help -@endcode - -@subsection cli_ex1_fmkstut FastMKS with a linear kernel on one dataset - -If only one dataset is specified (with \c -r or \c --reference_file), the -reference dataset is taken to be both the query and reference datasets. The -example below finds the 4 maximum kernels of each point in dataset.csv, using -the default linear kernel. - -@code -$ mlpack_fastmks -r dataset.csv -k 4 -v -p products.csv -i indices.csv -@endcode - -When the operation completes, the values of the kernels are saved in -products.csv and the indices of the points which give the maximum kernels are -saved in indices.csv. - -@code -$ head indices.csv -762,910,863,890 -762,910,426,568 -910,762,863,426 -762,910,863,426 -863,910,614,762 -762,863,910,614 -762,910,488,568 -762,910,863,426 -910,762,863,426 -863,762,910,614 -@endcode - -@code -$ head products.csv -1.6221652894e+00,1.5998743443e+00,1.5898890769e+00,1.5406789753e+00 -1.3387953449e+00,1.3317349486e+00,1.2966613184e+00,1.2774493620e+00 -1.6386110476e+00,1.6332029753e+00,1.5952629124e+00,1.5887195330e+00 -1.0917545803e+00,1.0820878726e+00,1.0668992636e+00,1.0419838050e+00 -1.2272441028e+00,1.2169643942e+00,1.2104597963e+00,1.2067780154e+00 -1.5720962456e+00,1.5618504956e+00,1.5609069923e+00,1.5235605095e+00 -1.3655478674e+00,1.3548593212e+00,1.3311547298e+00,1.3250728881e+00 -2.0119149744e+00,2.0043668067e+00,1.9847289214e+00,1.9298280046e+00 -1.1586923205e+00,1.1494586097e+00,1.1274872962e+00,1.1248172766e+00 -4.4789820372e-01,4.4618539778e-01,4.4200024852e-01,4.3989721792e-01 -@endcode - -We can see in this example that for point 0, the point with maximum kernel value -is point 762, with a kernel value of 1.622165. For point 3, the point with -third largest kernel value is point 863, with a kernel value of 1.0669. - -@subsection cli_ex2_fmkstut FastMKS on a reference and query dataset - -The query points may be different than the reference points. To specify a -different query set, the \c -q (or \c --query_file) option is used, as in the -example below. - -@code -$ mlpack_fastmks -q query_set.csv -r reference_set.csv -k 5 -i indices.csv \ -> -p products.csv -@endcode - -@subsection cli_ex3_fmkstut FastMKS with a different kernel - -The \c mlpack_fastmks program offers more than just the linear kernel. Valid -options are \c 'linear', \c 'polynomial', \c 'cosine', \c 'gaussian', -\c 'epanechnikov', \c 'triangular' and \c 'hyptan' (the hyperbolic tangent -kernel). Note that the hyperbolic tangent kernel is provably not a Mercer -kernel but is positive semidefinite on most datasets and is commonly used as a -kernel. Note also that the Gaussian kernel and other shift-invariant kernels -give the same results as nearest neighbor search (see \ref nstutorial). - -The kernel to use is specified with the \c -K (or \c --kernel) option. The -example below uses the cosine similarity as a kernel. - -@code -$ mlpack_fastmks -r dataset.csv -k 5 -K cosine -i indices.csv -p products.csv -v -@endcode - -@subsection cli_ex4_fmkstut Using single-tree search or naive search - -In some cases, it may be useful to not use the dual-tree FastMKS algorithm. -Instead you can specify the \c --single option, indicating that a tree should be -built only on the reference set, and then the queries should be processed in a -linear scan (instead of in a tree). Alternately, the \c -N (or \c --naive) -option makes the program not build trees at all and instead use brute-force -search to find the solutions. - -The example below uses single-tree search on two datasets with the linear -kernel. - -@code -$ mlpack_fastmks -q query_set.csv -r reference_set.csv --single -k 5 \ -> -p products.csv -i indices.csv -K linear -@endcode - -The example below uses naive search on one dataset. - -@code -$ mlpack_fastmks -r reference_set.csv -k 5 -N -p products.csv -i indices.csv -@endcode - -@subsection cli_ex5_fmkstut Parameters for alternate kernels - -Many of the alternate kernel choices have parameters which can be chosen; these -are detailed in this section. - - - \b \c -w (\c --bandwidth): this sets the bandwidth of the kernel, and is - applicable to the \c 'gaussian', \c 'epanechnikov', and \c 'triangular' - kernels. This is the "spread" of the kernel. - - - \b \c -d (\c --degree): this sets the degree of the polynomial kernel (the - power to which the result is raised). It is only applicable to the \c - 'polynomial' kernel. - - - \b \c -o (\c --offset): this sets the offset of the kernel, for the \c - 'polynomial' and \c 'hyptan' kernel. See \ref - mlpack::kernel::PolynomialKernel "the polynomial kernel documentation" and - \ref mlpack::kernel::HyperbolicTangentKernel - "the hyperbolic tangent kernel documentation" for more information. - - - \b \c -s (\c --scale): this sets the scale of the kernel, and is only - applicable to the \c 'hyptan' kernel. See \ref - mlpack::kernel::HyperbolicTangentKernel - "the hyperbolic tangent kernel documentation" for more information. - -@subsection cli_ex6_fmkstut Saving a FastMKS model/tree - -The \c mlpack_fastmks program also supports saving a model built on a reference -dataset (this model includes the tree, the kernel, and the search parameters). -The \c --output_model_file or \c -M option allows one to save these parameters -to disk for later usage. An example is below: - -@code -$ mlpack_fastmks -r reference_set.csv -K cosine -M fastmks_model.xml -@endcode - -This example builds a tree on the dataset in \c reference_set.csv using the -cosine similarity kernel, and saves the resulting model to \c fastmks_model.xml. -This model may then be used in later calls to the \c mlpack_fastmks program. - -@subsection cli_ex7_fmkstut Loading a FastMKS model for further searches - -Supposing that a FastMKS model has been saved with the \c --output_model_file or -\c -M parameter, that model can then be later loaded in subsequent calls to the -\c mlpack_fastmks program, using the \c --input_model_file or \c -m option. For -instance, with a model saved in \c fastmks_model.xml and a query set in -\c query_set.csv, we can find 3 max-kernel candidates, saving to \c indices.csv -and \c kernels.csv: - -@code -$ mlpack_fastmks -m fastmks_model.xml -k 3 -i indices.csv -p kernels.csv -@endcode - -Loading a model as opposed to building a model is advantageous because the -reference tree is already built. So, among other situations, this could be -useful in the setting where many different query sets (or many different values -of k) will be used. - -Note that the kernel cannot be changed in a saved model without rebuilding the -model entirely. - -@section fastmks_fmkstut The 'FastMKS' class - -The \c FastMKS<> class offers a simple API for use within C++ applications, and -allows further flexibility in kernel choice and tree type choice. However, -\c FastMKS<> has no default template parameter for the kernel type -- that must -be manually specified. Choices that \b mlpack provides include: - - - \ref mlpack::kernel::LinearKernel - - \ref mlpack::kernel::PolynomialKernel - - \ref mlpack::kernel::CosineDistance - - \ref mlpack::kernel::GaussianKernel - - \ref mlpack::kernel::EpanechnikovKernel - - \ref mlpack::kernel::TriangularKernel - - \ref mlpack::kernel::HyperbolicTangentKernel - - \ref mlpack::kernel::LaplacianKernel - - \ref mlpack::kernel::PSpectrumStringKernel - -The following examples use kernels from that list. Writing your own kernel is -detailed in \ref writing_kernel_fmkstut "the next section". Remember that when -you are using the C++ interface, the data matrices must be column-major. See -\ref matrices for more information. - -@subsection fastmks_ex1_fmkstut FastMKS on one dataset - -Given only a reference dataset, the following code will run FastMKS with k set -to 5. - -@code -#include -#include - -using namespace mlpack::fastmks; - -// The reference dataset, which is column-major. -extern arma::mat data; - -// This will initialize the FastMKS object with the linear kernel with default -// options: K(x, y) = x^T y. The tree is built in the constructor. -FastMKS f(data); - -// The results will be stored in these matrices. -arma::Mat indices; -arma::mat products; - -// Run FastMKS. -f.Search(5, indices, products); -@endcode - -@subsection fastmks_ex2_fmkstut FastMKS with a query and reference dataset - -In this setting we have both a query and reference dataset. We search for 10 -maximum kernels. - -@code -#include -#include - -using namespace mlpack::fastmks; -using namespace mlpack::kernel; - -// The reference and query datasets, which are column-major. -extern arma::mat referenceData; -extern arma::mat queryData; - -// This will initialize the FastMKS object with the triangular kernel with -// default options (bandwidth of 1). The reference tree is built in the -// constructor. -FastMKS f(referenceData); - -// The results will be stored in these matrices. -arma::Mat indices; -arma::mat products; - -// Run FastMKS. The query tree is built during the call to Search(). -f.Search(queryData, 10, indices, products); -@endcode - -@subsection fastmks_ex3_fmkstut FastMKS with an initialized kernel - -Often, kernels have parameters which need to be specified. \c FastMKS<> has -constructors which take initialized kernels. Note that temporary kernels cannot -be passed as an argument. The example below initializes a \c PolynomialKernel -object and then runs FastMKS with a query and reference dataset. - -@code -#include -#include - -using namespace mlpack::fastmks; -using namespace mlpack::kernel; - -// The reference and query datasets, which are column-major. -extern arma::mat referenceData; -extern arma::mat queryData; - -// Initialize the polynomial kernel with degree of 3 and offset of 2.5. -PolynomialKernel pk(3.0, 2.5); - -// Create the FastMKS object with the initialized kernel. -FastMKS f(referenceData, pk); - -// The results will be stored in these matrices. -arma::Mat indices; -arma::mat products; - -// Run FastMKS. -f.Search(queryData, 10, indices, products); -@endcode - -The syntax for running FastMKS with one dataset and an initialized kernel is -very similar: - -@code -f.Search(10, indices, products); -@endcode - -@subsection fastmks_ex4_fmkstut FastMKS with an already-created tree - -By default, \c FastMKS<> uses the cover tree datastructure (see \ref -mlpack::tree::CoverTree). Sometimes, it is useful to modify the parameters of -the cover tree. In this scenario, a tree must be built outside of the -constructor, and then passed to the appropriate \c FastMKS<> constructor. An -example on just a reference dataset is shown below, where the base of the cover -tree is modified. - -We also use an instantiated kernel, but because we are building our own tree, we -must use \ref mlpack::metric::IPMetric "IPMetric" so that our tree is built on -the metric induced by our kernel function. - -@code -#include -#include - -// The reference dataset, which is column-major. -extern arma::mat data; - -// Initialize the polynomial kernel with a degree of 4 and offset of 2.0. -PolynomialKernel pk(4.0, 2.0); - -// Create the metric induced by this kernel (because a kernel is not a metric -// and we can't build a tree on a kernel alone). -IPMetric metric(pk); - -// Now build a tree on the reference dataset using the instantiated metric and -// the custom base of 1.5 (default is 1.3). We have to be sure to use the right -// type here -- FastMKS needs the FastMKSStat object as the tree's -// StatisticType. -typedef tree::CoverTree, tree::FirstPointIsRoot, - FastMKSStat> TreeType; // Convenience typedef. -TreeType* tree = new TreeType(data, metric, 1.5); - -// Now initialize FastMKS with that statistic. We don't need to specify the -// TreeType template parameter since we are still using the default. We don't -// need to pass the kernel because that is contained in the tree. -FastMKS f(tree); - -// The results will be stored in these matrices. -arma::Mat indices; -arma::mat products; - -// Run FastMKS. -f.Search(10, indices, products); -@endcode - -The syntax is similar for the case where different query and reference datasets -are given; but trees for both need to be built in the manner specified above. -Be sure to build both trees using the same metric (or at least a metric with the -exact same parameters). - -@code -f.Search(queryTree, 10, indices, products); -@endcode - -@section writing_kernel_fmkstut Writing a custom kernel for FastMKS - -While \b mlpack provides some number of kernels in the mlpack::kernel namespace, -it is easy to create a custom kernel. To satisfy the KernelType policy, a class -must implement the following methods: - -@code -// Empty constructor is required. -KernelType(); - -// Evaluate the kernel between two points. -template -double Evaluate(const VecType& a, const VecType& b); -@endcode - -The template parameter \c VecType is helpful (but not necessary) so that the -kernel can be used with both sparse and dense matrices (\c arma::sp_mat and \c -arma::mat). - -@section custom_tree_fmkstut Using other tree types for FastMKS - -The use of the cover tree (see \ref mlpack::tree::CoverTree "CoverTree") is not -necessary for FastMKS, although it is the default tree type. A different type -of tree can be specified with the TreeType template parameter. However, the -tree type is required to have \ref mlpack::fastmks::FastMKSStat "FastMKSStat" as -the StatisticType, and for FastMKS to work, the tree must be built only on -kernel evaluations (or distance evaluations in the kernel space via -\ref mlpack::metric::IPMetric "IPMetric::Evaluate()"). - -Below is an example where a custom tree class, \c CustomTree, is used as the -tree type for FastMKS. In this example FastMKS is only run on one dataset. - -@code -#include -#include "custom_tree.hpp" - -using namespace mlpack::fastmks; -using namespace mlpack::tree; - -// The dataset that FastMKS will be run on. -extern arma::mat data; - -// The custom tree type. We'll assume that the first template parameter is the -// statistic type. -typedef CustomTree TreeType; - -// The FastMKS constructor will create the tree. -FastMKS f(data); - -// These will hold the results. -arma::Mat indices; -arma::mat products; - -// Run FastMKS. -f.Search(5, indices, products); -@endcode - -@section objects_fmkstut Running FastMKS on objects - -FastMKS has a lot of utility on objects which are not representable in some sort -of metric space. These objects might be strings, graphs, models, or other -objects. For these types of objects, questions based on distance don't really -make sense. One good example is with strings. The question "how far is 'dog' -from 'Taki Inoue'?" simply doesn't make sense. We can't have a centroid of the -terms 'Fritz', 'E28', and 'popsicle'. - -However, what we can do is define some sort of kernel on these objects. These -kernels generally correspond to some similarity measure, with one example being -the p-spectrum string kernel (see \ref mlpack::kernel::PSpectrumStringKernel). -Using that, we can say "how similar is 'dog' to 'Taki Inoue'?" and get an actual -numerical result by evaluating K('dog', 'Taki Inoue') (where K is our p-spectrum -string kernel). - -The only requirement on these kernels is that they are positive definite kernels -(or Mercer kernels). For more information on those details, refer to the -FastMKS paper. - -Remember that FastMKS is a tree-based method. But trees like the binary space -tree require centroids -- and as we said earlier, centroids often don't make -sense with these types of objects. Therefore, we need a type of tree which is -built \b exclusively on points in the dataset -- those are points which we can -evaluate our kernel function on. The cover tree is one example of a type of -tree satisfying this condition; its construction will only call the kernel -function on two points that are in the dataset. - -But, we have one more problem. The \c CoverTree class is built on \c arma::mat -objects (dense matrices). Our objects, however, are not necessarily -representable in a column of a matrix. To use the example we have been using, -strings cannot be represented easily in a matrix because they may all have -different lengths. - -The way to work around this problem is to create a "fake" data matrix which -simply holds indices to objects. A good example of how to do this is detailed -in the documentation for the \ref mlpack::kernel::PSpectrumStringKernel -"PSpectrumStringKernel". - -In short, the trick is to make each data matrix one-dimensional and containing -linear indices: - -@code -arma::mat data = "0 1 2 3 4 5 6 7 8"; -@endcode - -Then, when \c Evaluate() is called on the kernel function, the parameters will -be two one-dimensional vectors that simply contain indices to objects. The -example below details the process a little better: - -@code -// This function evaluates the kernel on two Objects (in this example, its -// implementation is not important; the only important thing is that the -// function exists). -double ObjectKernel::Evaluate(const Object& a, const Object& b) const; - -template -double ObjectKernel::Evaluate(const VecType& a, const VecType& b) const -{ - // Extract the indices from the vectors. - const size_t indexA = size_t(a[0]); - const size_t indexB = size_t(b[0]); - - // Assume that 'objects' is an array (or std::vector or other container) - // holding Objects. - const Object& objectA = objects[indexA]; - const Object& objectB = objects[indexB]; - - // Now call the function that does the actual evaluation on the objects and - // return its result. - return Evaluate(objectA, objectB); -} -@endcode - -As written earlier, the documentation for \ref -mlpack::kernel::PSpectrumStringKernel "PSpectrumStringKernel" is a good place to -consult for further reference on this. That kernel uses two dimensional -indices; one dimension represents the index of the string, and the other -represents whether it is referring to the query set or the reference set. If -your kernel is meant to work on separate query and reference sets, that strategy -should be considered. - -@section further_doc_fmkstut Further documentation - -For further documentation on the FastMKS class, consult the \ref -mlpack::fastmks::FastMKS "complete API documentation". - -*/ diff --git a/doc/tutorials/image.md b/doc/tutorials/image.md new file mode 100644 index 0000000000..dc08a085dd --- /dev/null +++ b/doc/tutorials/image.md @@ -0,0 +1,179 @@ +# Image Utilities Tutorial + +Image datasets are becoming increasingly popular in deep learning. + +mlpack's image saving/loading functionality is based on [stb/](https://github.com/nothings/stb). + +## Model API + +Image utilities supports loading and saving of images. + +It supports filetypes `jpg`, `png`, `tga`, `bmp`, `psd`, `gif`, `hdr`, `pic`, +`pnm` for loading and `jpg`, `png`, `tga`, `bmp`, `hdr` for saving. + +The datatype associated is unsigned char to support RGB values in the range +1-255. To feed data into the network typecast of `arma::Mat` may be required. +Images are stored in matrix as `(width * height * channels, numberOfImages)`. +Therefore `imageMatrix.col(0)` would be the first image if images are loaded in +`imageMatrix`. + +## `ImageInfo` + +The `ImageInfo` class contains the metadata of the images. + +```c++ +/** + * Instantiate the ImageInfo object with the image width, height, channels. + * + * @param width Image width. + * @param height Image height. + * @param channels number of channels in the image. + */ +ImageInfo(const size_t width, + const size_t height, + const size_t channels); +``` + +Other public members include the quality compression of the image if saved as +`jpg` (0-100). + +## Loading + +Standalone loading of images can be done with the function below. + +```c++ +/** + * Load the image file into the given matrix. + * + * @param filename Name of the image file. + * @param matrix Matrix to load the image into. + * @param info An object of ImageInfo class. + * @param fatal If an error should be reported as fatal (default false). + * @param transpose If true, flips the image, same as transposing the + * matrix after loading. + * @return Boolean value indicating success or failure of load. + */ +template +bool Load(const std::string& filename, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +``` + +Loading a test image is shown below. It also fills up the `ImageInfo` class +object. + +```c++ +data::ImageInfo info; +data::Load("test_image.png", matrix, info, false, true); +``` + +`ImageInfo` requires height, width, number of channels of the image. + +```c++ +size_t height = 64, width = 64, channels = 1; +data::ImageInfo info(width, height, channels); +``` + +More than one image can be loaded into the same matrix. + +Loading multiple images can be done using the function below. + +```c++ +/** + * Load the image file into the given matrix. + * + * @param files A vector consisting of filenames. + * @param matrix Matrix to save the image from. + * @param info An object of ImageInfo class. + * @param fatal If an error should be reported as fatal (default false). + * @param transpose If true, flips the image, same as transposing the + * matrix after loading. + * @return Boolean value indicating success or failure of load. + */ +template +bool Load(const std::vector& files, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +``` + +```c++ +data::ImageInfo info; +std::vector> files{"test_image1.bmp","test_image2.bmp"}; +data::load(files, matrix, info, false, true); +``` + +## Saving + +Saving images expects a matrix of type unsigned char in the form `(width * +height * channels, NumberOfImages)`. Just like loading, it can be used to save +one image or multiple images. Besides image data it also expects the shape of +the image as input `(width, height, channels)`. + +Saving one image can be done with the function below: + +```c++ +/** + * Save the image file from the given matrix. + * + * @param filename Name of the image file. + * @param matrix Matrix to save the image from. + * @param info An object of ImageInfo class. + * @param fatal If an error should be reported as fatal (default false). + * @param transpose If true, flips the image, same as transposing the + * matrix after loading. + * @return Boolean value indicating success or failure of load. + */ +template +bool Save(const std::string& filename, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +``` + +```c++ +data::ImageInfo info; +info.width = info.height = 25; +info.channels = 3; +info.quality = 90; +data::Save("test_image.bmp", matrix, info, false, true); +``` + +If the matrix contains more than one image, only the first one is saved. + +Saving multiple images can be done with the function below. + +```c++ +/** + * Save the image file from the given matrix. + * + * @param files A vector consisting of filenames. + * @param matrix Matrix to save the image from. + * @param info An object of ImageInfo class. + * @param fatal If an error should be reported as fatal (default false). + * @param transpose If true, Flips the image, same as transposing the + * matrix after loading. + * @return Boolean value indicating success or failure of load. + */ +template +bool Save(const std::vector& files, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +``` + +```c++ +data::ImageInfo info; +info.width = info.height = 25; +info.channels = 3; +info.quality = 90; +std::vector> files{"test_image1.bmp", "test_image2.bmp"}; +data::Save(files, matrix, info, false, true); +``` + +Multiple images are saved according to the vector of filenames specified. diff --git a/doc/tutorials/image/image.txt b/doc/tutorials/image/image.txt deleted file mode 100644 index 54fdf37f1e..0000000000 --- a/doc/tutorials/image/image.txt +++ /dev/null @@ -1,188 +0,0 @@ -/*! -@file image.txt -@author Mehul Kumar Nirala -@brief Tutorial for how to load and save images in mlpack. - -@page imagetutorial Image Utilities tutorial - -@section intro_imagetu Introduction - -Image datasets are becoming increasingly popular in deep learning. - -mlpack's image saving/loading functionality is based on [stb/](https://github.com/nothings/stb). - -@section toc_imagetu Table of Contents - -This tutorial is split into the following sections: - - - \ref intro_imagetu - - \ref toc_imagetu - - \ref model_api_imagetu - - \ref imageinfo_api_imagetu - - \ref load_api_imagetu - - \ref save_api_imagetu - -@section model_api_imagetu Model API - -Image utilities supports loading and saving of images. - -It supports filetypes "jpg", "png", "tga","bmp", "psd", "gif", "hdr", "pic", "pnm" for loading and "jpg", "png", "tga", "bmp", "hdr" for saving. - -The datatype associated is unsigned char to support RGB values in the range 1-255. To feed data into the network typecast of `arma::Mat` may be required. Images are stored in matrix as (width * height * channels, NumberOfImages). Therefore imageMatrix.col(0) would be the first image if images are loaded in imageMatrix. - -@section imageinfo_api_imagetu ImageInfo - -ImageInfo class contains the metadata of the images. -@code - /** - * Instantiate the ImageInfo object with the image width, height, channels. - * - * @param width Image width. - * @param height Image height. - * @param channels number of channels in the image. - */ - ImageInfo(const size_t width, - const size_t height, - const size_t channels); -@endcode -Other public memebers include: - - quality Compression of the image if saved as jpg (0-100). - -@section load_api_imagetu Load - - -Standalone loading of images. -@code - /** - * Load the image file into the given matrix. - * - * @param filename Name of the image file. - * @param matrix Matrix to load the image into. - * @param info An object of ImageInfo class. - * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, flips the image, same as transposing the - * matrix after loading. - * @return Boolean value indicating success or failure of load. - */ - template - bool Load(const std::string& filename, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal, - const bool transpose); -@endcode - -Loading a test image. It also fills up the ImageInfo class object. -@code -data::ImageInfo info; -data::Load("test_image.png", matrix, info, false, true); -@endcode - -ImageInfo requires height, width, number of channels of the image. - -@code -size_t height = 64, width = 64, channels = 1; -data::ImageInfo info(width, height, channels); -@endcode - -More than one image can be loaded into the same matrix. - -Loading multiple images: - -@code - /** - * Load the image file into the given matrix. - * - * @param files A vector consisting of filenames. - * @param matrix Matrix to save the image from. - * @param info An object of ImageInfo class. - * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, flips the image, same as transposing the - * matrix after loading. - * @return Boolean value indicating success or failure of load. - */ - template - bool Load(const std::vector& files, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal, - const bool transpose); -@endcode - -@code - data::ImageInfo info; - std::vector> files{"test_image1.bmp","test_image2.bmp"}; - data::load(files, matrix, info, false, true); -@endcode - -@section save_api_imagetu Save - -Save images expects a matrix of type unsigned char in the form (width * height * channels, NumberOfImages). -Just like load it can be used to save one image or multiple images. Besides image data it also expects the shape of the image as input (width, height, channels). - -Saving one image: - -@code - /** - * Save the image file from the given matrix. - * - * @param filename Name of the image file. - * @param matrix Matrix to save the image from. - * @param info An object of ImageInfo class. - * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, flips the image, same as transposing the - * matrix after loading. - * @return Boolean value indicating success or failure of load. - */ - template - bool Save(const std::string& filename, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal, - const bool transpose); -@endcode - -@code - data::ImageInfo info; - info.width = info.height = 25; - info.channels = 3; - info.quality = 90; - data::Save("test_image.bmp", matrix, info, false, true); -@endcode - -If the matrix contains more than one image, only the first one is saved. - -Saving multiple images: - -@code - /** - * Save the image file from the given matrix. - * - * @param files A vector consisting of filenames. - * @param matrix Matrix to save the image from. - * @param info An object of ImageInfo class. - * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, Flips the image, same as transposing the - * matrix after loading. - * @return Boolean value indicating success or failure of load. - */ - template - bool Save(const std::vector& files, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal, - const bool transpose); -@endcode - -@code - data::ImageInfo info; - info.width = info.height = 25; - info.channels = 3; - info.quality = 90; - std::vector> files{"test_image1.bmp", "test_image2.bmp"}; - data::Save(files, matrix, info, false, true); -@endcode - -Multiple images are saved according to the vector of filenames specified. - -*/ diff --git a/doc/tutorials/kmeans.md b/doc/tutorials/kmeans.md new file mode 100644 index 0000000000..1c4a852a22 --- /dev/null +++ b/doc/tutorials/kmeans.md @@ -0,0 +1,647 @@ +# K-Means Tutorial + +The popular k-means algorithm for clustering has been around since the late +1950s, and the standard algorithm was proposed by Stuart Lloyd in 1957. Given a +set of points `X`, k-means clustering aims to partition each point `x_i` into a +cluster `c_j` (where `j <= k` and `k`, the number of clusters, is a parameter). +The partitioning is done to minimize the objective function + +``` +sum_j^k sum_{x_i in c_j} || x_i - m_j ||^2 +``` + +where `m_j` is the centroid of cluster `c_j`. The standard algorithm +is a two-step algorithm: + + - *Assignment* step. Each point `x_i` in `X` is assigned to the cluster whose + centroid it is closest to. + + - *Update* step. Using the new cluster assignments, the centroids of each + cluster are recalculated. + +The algorithm has converged when no more assignment changes are happening with +each iteration. However, this algorithm can get stuck in local minima of the +objective function and is particularly sensitive to the initial cluster +assignments. Also, situations can arise where the algorithm will never converge +but reaches steady state---for instance, one point may be changing between two +cluster assignments. + +There is vast literature on the k-means algorithm and its uses, as well as +strategies for choosing initial points effectively and keeping the algorithm +from converging in local minima. mlpack does implement some of these, notably +the Bradley-Fayyad algorithm (see the reference below) for choosing refined +initial points. Importantly, the C++ `KMeans` class makes it very easy to +improve the k-means algorithm in a modular way. + +```c++ +@inproceedings{bradley1998refining, + title={Refining initial points for k-means clustering}, + author={Bradley, Paul S. and Fayyad, Usama M.}, + booktitle={Proceedings of the Fifteenth International Conference on Machine + Learning (ICML 1998)}, + volume={66}, + year={1998} +} +``` + +mlpack provides: + + - a simple command-line executable to run k-means + - a simple C++ interface to run k-means + - a generic, extensible, and powerful C++ class for complex usage + +## Command-line `mlpack_kmeans + +mlpack provides a command-line executable, `mlpack_kmeans`, to allow easy +execution of the k-means algorithm on data. Complete documentation of the +executable can be found by typing + +```sh +$ mlpack_kmeans --help +``` + +Note that mlpack also has bindings to other languages and provides, e.g., the +`kmeans()` function in Python that is very similar to the `mlpack_kmeans` +command-line program. So each example below can be easily adapted to another +language. + +Below are several examples demonstrating simple use of the `mlpack_kmeans` +executable. + +### Simple k-means clustering + +We want to find 5 clusters using the points in the file `dataset.csv`. By +default, if any of the clusters end up empty, that cluster will be reinitialized +to contain the point furthest from the cluster with maximum variance. The +cluster assignments of each point will be stored in `assignments.csv`. Each row +in assignments.csv will correspond to the row in `dataset.csv`. + +```sh +$ mlpack_kmeans -c 5 -i dataset.csv -v -o assignments.csv +``` + +### Saving the resulting centroids + +Sometimes it is useful to save the centroids of the clusters found by k-means; +one example might be for plotting the points. The `-C` (`--centroid_file`) +option allows specification of a file into which the centroids will be saved +(one centroid per line, if it is a CSV or other text format). + +```sh +$ mlpack_kmeans -c 5 -i dataset.csv -v -o assignments.csv -C centroids.csv +``` + +### Allowing empty clusters + +If you would like to allow empty clusters to exist, instead of reinitializing +them, simply specify the `-e` (`--allow_empty_clusters`) option. Note that when +you save your clusters, even empty clusters will still have centroids. The +centroids of the empty cluster will be the same as what they were on the last +iteration when the cluster was not empty. + +```sh +$ mlpack_kmeans -c 5 -i dataset.csv -v -e -o assignments.csv -C centroids.csv +``` + +### Killing empty clusters + +If you would like to kill empty clusters, instead of reinitializing them, simply +specify the `-E` (`--kill_empty_clusters`) option. Note that when you save your +clusters, all the empty clusters will be removed and the final result may +contain less than specified number of clusters. + +```sh +$ mlpack_kmeans -c 5 -i dataset.csv -v -E -o assignments.csv -C centroids.csv +``` + +### Limiting the maximum number of iterations + +As mentioned earlier, the k-means algorithm can often fail to converge. In such +a situation, it may be useful to stop the algorithm by way of limiting the +maximum number of iterations. This can be done with the `-m` +(`--max_iterations`) parameter, which is set to 1000 by default. If the maximum +number of iterations is 0, the algorithm will run until convergence---or +potentially forever. The example below sets a maximum of 250 iterations. + +```sh +$ mlpack_kmeans -c 5 -i dataset.csv -v -o assignments.csv -m 250 +``` + +### Using Bradley-Fayyad 'refined start' + +The method proposed by Bradley and Fayyad in their paper "Refining initial +points for k-means clustering" is implemented in mlpack. This strategy samples +points from the dataset and runs k-means clustering on those points multiple +times, saving the resulting clusters. Then, k-means clustering is run on those +clusters, yielding the original number of clusters. The centroids of those +resulting clusters are used as initial centroids for k-means clustering on the +entire dataset. + +This technique generally gives better initial points than the default random +partitioning, but depending on the parameters, it can take much longer. This +initialization technique is enabled with the `-r` (`--refined_start`) option. +The `-S` (`--samplings`) parameter controls how many samplings of the dataset +are performed, and the `-p` (`--percentage`) parameter controls how much of the +dataset is randomly sampled for each sampling (it must be between 0.0 and 1.0). +For more information on the refined start technique, see the paper referenced in +the introduction of this tutorial. + +The example below performs k-means clustering, giving 5 clusters, using the +refined start technique, sampling 10% of the dataset 25 times to produce the +initial centroids. + +```sh +$ mlpack_kmeans -c 5 -i dataset.csv -v -o assignments.csv -r -S 25 -p 0.2 +``` + +### Using different k-means algorithms + +The `mlpack_kmeans` program implements six different strategies for clustering; +each of these gives the exact same results, but will have different runtimes. +The particular algorithm to use can be specified with the `-a` or `--algorithm` +option. The choices are: + + - `naive`: the standard Lloyd iteration; takes `O(kN)` time per iteration. + - `pelleg-moore`: the 'blacklist' algorithm, which builds a kd-tree on the + data. This can be fast when k is small and the dimensionality is reasonably + low. + - `elkan`: Elkan's algorithm for k-means, which maintains upper and lower + distance bounds between each point and each centroid. This can be very fast, + but it does not scale well to the case of large N or k, and uses a lot of + memory. + - `hamerly`: Hamerly's algorithm is a variant of Elkan's algorithm that + handles memory usage much better and thus can operate with much larger + datasets than Elkan's algorithm. + - `dualtree`: The dual-tree algorithm for k-means builds a kd-tree on both the + centroids and the points in order to prune away as much work as possible. + This algorithm is most effective when both N and k are large. + - `dualtree-covertree`: This is the dual-tree algorithm using cover trees + instead of kd-trees. It satisfies the runtime guarantees specified in the + dual-tree k-means paper. + +In general, the `naive` algorithm will be much slower than the others on +datasets that are larger than tiny. + +The example below uses the `dualtree` algorithm to perform k-means clustering +with 5 clusters on the dataset in `dataset.csv`, using the initial centroids in +`initial_centroids.csv`, saving the resulting cluster assignments to +`assignments.csv`: + +```sh +$ mlpack_kmeans -i dataset.csv -c 5 -v -I initial_centroids.csv -a dualtree \ +> -o assignments.csv +``` + +## The `KMeans` class + +The `KMeans<>` class (with default template parameters) provides a simple way +to run k-means clustering using mlpack in C++. The default template +parameters for `KMeans<>` will initialize cluster assignments randomly and +disallow empty clusters. When an empty cluster is encountered, the point +furthest from the cluster with maximum variance is set to the centroid of the +empty cluster. + +### Running k-means and getting cluster assignments + +The simplest way to use the `KMeans<>` class is to pass in a dataset and a +number of clusters, and receive the cluster assignments in return. Note that +the dataset must be column-major---that is, one column corresponds to one point. +See [the matrices guide](../user/matrices.md) for more information. + +```c++ +#include + +using namespace mlpack::kmeans; + +// The dataset we are clustering. +extern arma::mat data; +// The number of clusters we are getting. +extern size_t clusters; + +// The assignments will be stored in this vector. +arma::Row assignments; + +// Initialize with the default arguments. +KMeans<> k; +k.Cluster(data, clusters, assignments); +``` + +Now, the vector `assignments` holds the cluster assignments of each point in the +dataset. + +### Running k-means and getting centroids of clusters + +Often it is useful to not only have the cluster assignments, but the centroids +of each cluster. Another overload of `Cluster()` makes this easily possible: + +```c++ +#include + +using namespace mlpack::kmeans; + +// The dataset we are clustering. +extern arma::mat data; +// The number of clusters we are getting. +extern size_t clusters; + +// The assignments will be stored in this vector. +arma::Row assignments; +// The centroids will be stored in this matrix. +arma::mat centroids; + +// Initialize with the default arguments. +KMeans<> k; +k.Cluster(data, clusters, assignments, centroids); +``` + +Note that the centroids matrix has columns equal to the number of clusters and +rows equal to the dimensionality of the dataset. Each column represents the +centroid of the according cluster---`centroids.col(0)` represents the centroid +of the first cluster. + +### Limiting the maximum number of iterations + +The first argument to the constructor allows specification of the maximum number +of iterations. This is useful because often, the k-means algorithm does not +converge, and is terminated after a number of iterations. Setting this +parameter to 0 indicates that the algorithm will run until convergence---note +that in some cases, convergence may never happen. The default maximum number of +iterations is 1000. + +```c++ +// The first argument is the maximum number of iterations. Here we set it to +// 500 iterations. +KMeans<> k(500); +``` + +Then you can run `Cluster()` as normal. + +### Setting initial cluster assignments + +If you have an initial guess for the cluster assignments for each point, you can +fill the assignments vector with the guess and then pass an extra boolean +(`initialAssignmentGuess`) as `true` to the `Cluster()` method. Below are +examples for either overload of `Cluster()`. + +```c++ +#include + +using namespace mlpack::kmeans; + +// The dataset we are clustering on. +extern arma::mat dataset; +// The number of clusters we are obtaining. +extern size_t clusters; + +// A vector pre-filled with initial assignment guesses. +extern arma::Row assignments; + +KMeans<> k; + +// The boolean set to true indicates that our assignments vector is filled with +// initial guesses. +k.Cluster(dataset, clusters, assignments, true); +``` + +```c++ +#include + +using namespace mlpack::kmeans; + +// The dataset we are clustering on. +extern arma::mat dataset; +// The number of clusters we are obtaining. +extern size_t clusters; + +// A vector pre-filled with initial assignment guesses. +extern arma::Row assignments; + +// This will hold the centroids of the finished clusters. +arma::mat centroids; + +KMeans<> k; + +// The boolean set to true indicates that our assignments vector is filled with +// initial guesses. +k.Cluster(dataset, clusters, assignments, centroids, true); +``` + +***Note***: If you have a heuristic or algorithm which makes initial guesses, a +more elegant solution is to create a new class fulfilling the +`InitialPartitionPolicy` template policy. See the section about changing the +initial partitioning strategy for more details. + +***Note***: If you set the `InitialPartitionPolicy` parameter to something other +than the default but give an initial cluster assignment guess, the +`InitialPartitionPolicy` will not be used to initialize the algorithm. See the +section about changing the initial partitioning strategy for more details. + +### Setting initial cluster centroids + +An equally important option to being able to make initial cluster assignment +guesses is to make initial cluster centroid guesses without having to assign +each point in the dataset to an initial cluster. This is similar to the +previous section, but now you must pass two extra booleans---the first +(`initialAssignmentGuess`) as `false`, indicating that there are not initial +cluster assignment guesses, and the second (`initialCentroidGuess`) as `true`, +indicating that the centroids matrix is filled with initial centroid guesses. + +This, of course, only works with the overload of `Cluster()` that takes a matrix +to put the resulting centroids in. Below is an example. + +```c++ +#include + +using namespace mlpack::kmeans; + +// The dataset we are clustering on. +extern arma::mat dataset; +// The number of clusters we are obtaining. +extern size_t clusters; + +// A matrix pre-filled with guesses for the initial cluster centroids. +extern arma::mat centroids; + +// This will be filled with the final cluster assignments for each point. +arma::Row assignments; + +KMeans<> k; + +// Remember, the first boolean indicates that we are not giving initial +// assignment guesses, and the second boolean indicates that we are giving +// initial centroid guesses. +k.Cluster(dataset, clusters, assignments, centroids, false, true); +``` + +***Note***: If you have a heuristic or algorithm which makes initial guesses, a +more elegant solution is to create a new class fulfilling the +`InitialPartitionPolicy` template policy. See the section about changing the +initial partitioning strategy for more details. + +***Note***: If you set the `InitialPartitionPolicy` parameter to something other +than the default but give an initial cluster centroid guess, the +`InitialPartitionPolicy` will not be used to initialize the algorithm. See the +section about changing the initial partitioning strategy for more details. + +### Running sparse k-means + +The `Cluster()` function can work on both sparse and dense matrices, so all of +the above examples can be used with sparse matrices instead, if the fifth +template parameter is modified. Below is a simple example. Note that the +centroids are returned as a dense matrix, because the centroids of collections +of sparse points are not generally sparse. + +```c++ +// The sparse dataset. +extern arma::sp_mat sparseDataset; +// The number of clusters. +extern size_t clusters; + +// The assignments will be stored in this vector. +arma::Row assignments; +// The centroids of each cluster will be stored in this sparse matrix. +arma::sp_mat sparseCentroids; + +// We must change the fifth (and last) template parameter. +KMeans k; +k.Cluster(sparseDataset, clusters, assignments, sparseCentroids); +``` + +### Template parameters for the `KMeans` class + +The `KMeans<>` class also takes three template parameters, which can be +modified to change the behavior of the k-means algorithm. There are three +template parameters: + + - `MetricType`: controls the distance metric used for clustering (by default, + the squared Euclidean distance is used) + - `InitialPartitionPolicy`: the method by which initial clusters are set; by + default, `SampleInitialization` is used + - `EmptyClusterPolicy`: the action taken when an empty cluster is encountered; + by default, `MaxVarianceNewCluster` is used + - `LloydStepType`: this defines the strategy used to make a single Lloyd + iteration; by default this is the typical Lloyd iteration specified in + `NaiveKMeans` + - `MatType`: type of data matrix to use for clustering + +The class is defined like below: + +```c++ +template< + typename DistanceMetric = mlpack::metric::SquaredEuclideanDistance, + typename InitialPartitionPolicy = SampleInitialization, + typename EmptyClusterPolicy = MaxVarianceNewCluster, + template class LloydStepType = NaiveKMeans, + typename MatType = arma::mat +> +class KMeans; +``` + +In the following sections, each policy is described further, with examples of +how to modify them. + +### Changing the distance metric used for k-means + +Most machine learning algorithms in mlpack support modifying the distance +metric, and `KMeans<>` is no exception. Similar to `NeighborSearch` (see the +section in the [NeighborSearch tutorial](neighbor_search.md)), any class in +`mlpack::metric` can be given as an argument. The `mlpack::metric::LMetric` +class is a good example implementation. + +A class fulfilling the [MetricType policy](../developer/metrictype.md) must +provide the following two functions: + +```c++ +// Empty constructor is required. +MetricType(); + +// Compute the distance between two points. +template +double Evaluate(const VecType& a, const VecType& b); +``` + +Most of the standard metrics that could be used are stateless and therefore the +`Evaluate()` method is implemented statically. However, there are metrics, such +as the Mahalanobis distance (`mlpack::metric::MahalanobisDistance`), that store +state. To this end, an instantiated `MetricType` object is stored within the +`KMeans` class. The example below shows how to pass an instantiated +`MahalanobisDistance` in the constructor. + +```c++ +// The initialized Mahalanobis distance. +extern mlpack::metric::MahalanobisDistance distance; + +// We keep the default arguments for the maximum number of iterations, but pass +// our instantiated metric. +KMeans k(1000, distance); +``` + +***Note***: While the `MetricType` policy only requires two methods, one of +which is an empty constructor, more can always be added. +`mlpack::metric::MahalanobisDistance` also has constructors with parameters, +because it is a stateful metric. + +### Changing the initial partitioning strategy used for k-means + +There have been many initial cluster strategies for k-means proposed in the +literature. Fortunately, the `KMeans<>` class makes it very easy to implement +one of these methods and plug it in without needing to modify the existing +algorithm code at all. + +By default, the `KMeans<>` class uses `mlpack::kmeans::SampleInitialization`, +which randomly samples points as initial centroids. However, writing a new +policy is simple; it needs to only implement the following functions: + +```c++ +// Empty constructor is required. +InitialPartitionPolicy(); + +// Only *one* of the following two functions is required! You should implement +// whichever you find more convenient to implement. + +// This function is called to initialize the clusters and returns centroids. +template +void Cluster(MatType& data, + const size_t clusters, + arma::mat& centroids); + +// This function is called to initialize the clusters and returns individual +// point assignments. The centroids will then be calculated from the given +// assignments. +template +void Cluster(MatType& data, + const size_t clusters, + arma::Row assignments); +``` + +The templatization of the `Cluster()` function allows both dense and sparse +matrices to be passed in. If the desired policy does not work with sparse (or +dense) matrices, then the method can be written specifically for one type of +matrix---however, be warned that if you try to use `KMeans` with that policy and +the wrong type of matrix, you will get many ugly compilation errors! + +```c++ +// The Cluster() function specialized for dense matrices. +void Cluster(arma::mat& data, + const size_t clusters, + arma::Row assignments); +``` + +Note that only one of the two possible `Cluster()` functions are required. This +is because sometimes it is easier to express an initial partitioning policy as +something that returns point assignments, and sometimes it is easier to express +the policy as something that returns centroids. The `KMeans<>` class will use +whichever of these two functions is given; if both are given, the overload that +returns centroids will be preferred. + +One alternate to the default `SampleInitialization` policy is the `RefinedStart` +policy, which is an implementation of the Bradley and Fayyad approach for +finding initial points detailed in "Refined initial points for k-means +clustering" and other places in this document. Another option is the +`RandomPartition class`, which randomly assigns points to clusters, but this may +not work very well for most settings. See the documentation for +`mlpack::kmeans::RefinedStart` and `mlpack::kmeans::RandomPartition` for more +information. + +If the `Cluster()` method returns point assignments instead of centroids, then +valid initial assignments must be returned for every point in the dataset. + +As with the `MetricType` template parameter, an initialized +`InitialPartitionPolicy` can be passed to the constructor of `KMeans` as a +fourth argument. + +### Changing the action taken when an empty cluster is encountered + +Sometimes, during clustering, a situation will arise where a cluster has no +points in it. The `KMeans` class allows easy customization of the action to be +taken when this occurs. By default, the point furthest from the centroid of the +cluster with maximum variance is taken as the centroid of the empty cluster; +this is implemented in the `mlpack::kmeans::MaxVarianceNewCluster` class. +Another alternate choice is the `mlpack::kmeans::AllowEmptyClusters` class, +which simply allows empty clusters to persist. + +A custom policy can be written and it must implement the following methods: + +```c++ +// Empty constructor is required. +EmptyClusterPolicy(); + +// This function is called when an empty cluster is encountered. emptyCluster +// indicates the cluster which is empty, and then the clusterCounts and +// assignments are meant to be modified by the function. The function should +// return the number of modified points. +template +size_t EmptyCluster(const MatType& data, + const size_t emptyCluster, + const MatType& centroids, + arma::Col& clusterCounts, + arma::Row& assignments); +``` + +The `EmptyCluster()` function is called for each cluster that is empty at each +iteration of the algorithm. As with `InitialPartitionPolicy`, the +`EmptyCluster()` function does not need to be generalized to support both dense +and sparse matrices---but usage with the wrong type of matrix will cause +compilation errors. + +Like the other template parameters to `KMeans`, `EmptyClusterPolicy` +implementations that have state can be passed to the constructor of `KMeans` as +a fifth argument. See the `kmeans::KMeans` documentation for further details. + +### The `LloydStepType` template parameter + +The internal algorithm used for a single step of the k-means algorithm can +easily be changed; mlpack implements several existing classes that satisfy +the `LloydStepType` policy: + + - `mlpack::kmeans::NaiveKMeans` + - `mlpack::kmeans::ElkanKMeans` + - `mlpack::kmeans::HamerlyKMeans` + - `mlpack::kmeans::PellegMooreKMeans` + - `mlpack::kmeans::DualTreeKMeans` + +Note that the `LloydStepType` policy is itself a template template parameter, +and must accept two template parameters of its own: + + - `MetricType`: the type of metric to use + - `MatType`: the type of data matrix to use + +The `LloydStepType` policy also mandates three functions: + + - a constructor: `LloydStepType(const MatType& dataset, MetricType& metric);` + - an `Iterate()` function: + +```c++ +/** + * Run a single iteration of the Lloyd algorithm, updating the given centroids + * into the newCentroids matrix. If any cluster is empty (that is, if any + * cluster has no points assigned to it), then the centroid associated with + * that cluster may be filled with invalid data (it will be corrected later). + * + * @param centroids Current cluster centroids. + * @param newCentroids New cluster centroids. + * @param counts Number of points in each cluster at the end of the iteration. + */ +double Iterate(const arma::mat& centroids, + arma::mat& newCentroids, + arma::Col& counts); +``` + + - a function to get the number of distance calculations: + +```c++ +size_t DistanceCalculations() const { return distanceCalculations; } +``` + +Note that `Iterate()` does not need to return valid centroids if the cluster is +empty. This is because `EmptyClusterPolicy` will handle the empty centroid. +This behavior can be used to avoid small amounts of computation. + +For examples, see the five aforementioned implementations of classes that +satisfy the `LloydStepType` policy. + +## Further documentation + +For further documentation on the `KMeans` class, consult the comments in the +source code, found in `mlpack/methods/kmeans/`. diff --git a/doc/tutorials/kmeans/kmeans.txt b/doc/tutorials/kmeans/kmeans.txt deleted file mode 100644 index 5a332c50e4..0000000000 --- a/doc/tutorials/kmeans/kmeans.txt +++ /dev/null @@ -1,698 +0,0 @@ -/*! - -@file kmeans.txt -@author Ryan Curtin -@brief Tutorial for how to use k-means in mlpack. - -@page kmtutorial K-Means tutorial (kmeans) - -@section intro_kmtut Introduction - -The popular k-means algorithm for clustering has been around since the late -1950s, and the standard algorithm was proposed by Stuart Lloyd in 1957. Given a -set of points \f$ X \f$, k-means clustering aims to partition each point \f$ x_i -\f$ into a cluster \f$ c_j \f$ (where \f$ j \le k \f$ and \f$ k \f$, the number -of clusters, is a parameter). The partitioning is done to minimize the -objective function - -\f[ -\sum_{j = 1}^{k} \sum_{x_i \in c_j} \| x_i - \mu_j \|^2 -\f] - -where \f$\mu_j\f$ is the centroid of cluster \f$c_j\f$. The standard algorithm -is a two-step algorithm: - - - \b Assignment \b step. Each point \f$x_i\f$ in \f$X\f$ is assigned to the - cluster whose centroid it is closest to. - - - \b Update \b step. Using the new cluster assignments, the centroids of each - cluster are recalculated. - -The algorithm has converged when no more assignment changes are happening with -each iteration. However, this algorithm can get stuck in local minima of the -objective function and is particularly sensitive to the initial cluster -assignments. Also, situations can arise where the algorithm will never converge -but reaches steady state -- for instance, one point may be changing between two -cluster assignments. - -There is vast literature on the k-means algorithm and its uses, as well as -strategies for choosing initial points effectively and keeping the algorithm -from converging in local minima. \b mlpack does implement some of these, -notably the Bradley-Fayyad algorithm (see the reference below) for choosing -refined initial points. Importantly, the C++ \c KMeans class makes it very easy -to improve the k-means algorithm in a modular way. - -@code -@inproceedings{bradley1998refining, - title={Refining initial points for k-means clustering}, - author={Bradley, Paul S. and Fayyad, Usama M.}, - booktitle={Proceedings of the Fifteenth International Conference on Machine - Learning (ICML 1998)}, - volume={66}, - year={1998} -} -@endcode - -\b mlpack provides: - - - a \ref cli_kmtut "simple command-line executable" to run k-means - - a \ref kmeans_kmtut "simple C++ interface" to run k-means - - a \ref kmeans_template_kmtut "generic, extensible, and powerful C++ class" - for complex usage - -@section toc_kmtut Table of Contents - -A list of all the sections this tutorial contains. - - - \ref intro_kmtut - - \ref toc_kmtut - - \ref cli_kmtut - - \ref cli_ex1_kmtut - - \ref cli_ex2_kmtut - - \ref cli_ex3_kmtut - - \ref cli_ex4_kmtut - - \ref cli_ex6_kmtut - - \ref cli_ex7_kmtut - - \ref kmeans_kmtut - - \ref kmeans_ex1_kmtut - - \ref kmeans_ex2_kmtut - - \ref kmeans_ex3_kmtut - - \ref kmeans_ex5_kmtut - - \ref kmeans_ex6_kmtut - - \ref kmeans_ex7_kmtut - - \ref kmeans_template_kmtut - - \ref kmeans_metric_kmtut - - \ref kmeans_initial_partition_kmtut - - \ref kmeans_empty_cluster_kmtut - - \ref kmeans_lloyd_kmtut - - \ref further_doc_kmtut - -@section cli_kmtut Command-Line 'kmeans' - -\b mlpack provides a command-line executable, \c mlpack_kmeans, to allow easy -execution of the k-means algorithm on data. Complete documentation of the -executable can be found by typing - -@code -$ mlpack_kmeans --help -@endcode - -As of October 2014, support for overclustering has been removed due to bugs and -lack of usage. If this is support you were using, or are interested, please -file a bug or get in touch with the \b mlpack developers in some way so that the -support can be re-implemented. - -Below are several examples demonstrating simple use of the \c mlpack_kmeans -executable. - -@subsection cli_ex1_kmtut Simple k-means clustering - -We want to find 5 clusters using the points in the file dataset.csv. By -default, if any of the clusters end up empty, that cluster will be reinitialized -to contain the point furthest from the cluster with maximum variance. The -cluster assignments of each point will be stored in assignments.csv. Each row -in assignments.csv will correspond to the row in dataset.csv. - -@code -$ mlpack_kmeans -c 5 -i dataset.csv -v -o assignments.csv -@endcode - -@subsection cli_ex2_kmtut Saving the resulting centroids - -Sometimes it is useful to save the centroids of the clusters found by k-means; -one example might be for plotting the points. The \c -C (\c --centroid_file) -option allows specification of a file into which the centroids will be saved -(one centroid per line, if it is a CSV or other text format). - -@code -$ mlpack_kmeans -c 5 -i dataset.csv -v -o assignments.csv -C centroids.csv -@endcode - -@subsection cli_ex3_kmtut Allowing empty clusters - -If you would like to allow empty clusters to exist, instead of reinitializing -them, simply specify the \c -e (\c --allow_empty_clusters) option. Note that -when you save your clusters, even empty clusters will still have centroids. -The centroids of the empty cluster will be the same as what they were on the -last iteration when the cluster was not empty. - -@code -$ mlpack_kmeans -c 5 -i dataset.csv -v -e -o assignments.csv -C centroids.csv -@endcode - -@subsection cli_ex3a_kmtut Killing empty clusters - -If you would like to kill empty clusters , instead of reinitializing -them, simply specify the \c -E (\c --kill_empty_clusters) option. Note that -when you save your clusters, all the empty clusters will be removed and the -final result may contain less than specified number of clusters. - -@code -$ mlpack_kmeans -c 5 -i dataset.csv -v -E -o assignments.csv -C centroids.csv -@endcode - -@subsection cli_ex4_kmtut Limiting the maximum number of iterations - -As mentioned earlier, the k-means algorithm can often fail to converge. In such -a situation, it may be useful to stop the algorithm by way of limiting the -maximum number of iterations. This can be done with the \c -m (\c ---max_iterations) parameter, which is set to 1000 by default. If the maximum -number of iterations is 0, the algorithm will run until convergence -- or -potentially forever. The example below sets a maximum of 250 iterations. - -@code -$ mlpack_kmeans -c 5 -i dataset.csv -v -o assignments.csv -m 250 -@endcode - -@subsection cli_ex6_kmtut Using Bradley-Fayyad "refined start" - -The method proposed by Bradley and Fayyad in their paper "Refining initial -points for k-means clustering" is implemented in \b mlpack. This strategy -samples points from the dataset and runs k-means clustering on those points -multiple times, saving the resulting clusters. Then, k-means clustering is run -on those clusters, yielding the original number of clusters. The centroids of -those resulting clusters are used as initial centroids for k-means clustering on -the entire dataset. - -This technique generally gives better initial points than the default random -partitioning, but depending on the parameters, it can take much longer. This -initialization technique is enabled with the \c -r (\c --refined_start) option. -The \c -S (\c --samplings) parameter controls how many samplings of the dataset -are performed, and the \c -p (\c --percentage) parameter controls how much of -the dataset is randomly sampled for each sampling (it must be between 0.0 and -1.0). For more information on the refined start technique, see the paper -referenced in the introduction of this tutorial. - -The example below performs k-means clustering, giving 5 clusters, using the -refined start technique, sampling 10% of the dataset 25 times to produce the -initial centroids. - -@code -$ mlpack_kmeans -c 5 -i dataset.csv -v -o assignments.csv -r -S 25 -p 0.2 -@endcode - -@subsection cli_ex7_kmtut Using different k-means algorithms - -The \c mlpack_kmeans program implements six different strategies for -clustering; each of these gives the exact same results, but will have different -runtimes. The particular algorithm to use can be specified with the \c -a or -\c --algorithm option. The choices are: - - - \c naive: the standard Lloyd iteration; takes \f$O(kN)\f$ time per iteration. - - \c pelleg-moore: the 'blacklist' algorithm, which builds a kd-tree on the - data. This can be fast when k is small and the dimensionality is reasonably - low. - - \c elkan: Elkan's algorithm for k-means, which maintains upper and lower - distance bounds between each point and each centroid. This can be very fast, - but it does not scale well to the case of large N or k, and uses a lot of - memory. - - \c hamerly: Hamerly's algorithm is a variant of Elkan's algorithm that - handles memory usage much better and thus can operate with much larger - datasets than Elkan's algorithm. - - \c dualtree: The dual-tree algorithm for k-means builds a kd-tree on both the - centroids and the points in order to prune away as much work as possible. - This algorithm is most effective when both N and k are large. - - \c dualtree-covertree: This is the dual-tree algorithm using cover trees - instead of kd-trees. It satisfies the runtime guarantees specified in the - dual-tree k-means paper. - -In general, the \c naive algorithm will be much slower than the others on -datasets that are larger than tiny. - -The example below uses the \c dualtree algorithm to perform k-means clustering -with 5 clusters on the dataset in \c dataset.csv, using the initial centroids in -\c initial_centroids.csv, saving the resulting cluster assignments to -\c assignments.csv: - -@code -$ mlpack_kmeans -i dataset.csv -c 5 -v -I initial_centroids.csv -a dualtree \ -> -o assignments.csv -@endcode - -@section kmeans_kmtut The 'KMeans' class - -The \c KMeans<> class (with default template parameters) provides a simple way -to run k-means clustering using \b mlpack in C++. The default template -parameters for \c KMeans<> will initialize cluster assignments randomly and -disallow empty clusters. When an empty cluster is encountered, the point -furthest from the cluster with maximum variance is set to the centroid of the -empty cluster. - -@subsection kmeans_ex1_kmtut Running k-means and getting cluster assignments - -The simplest way to use the \c KMeans<> class is to pass in a dataset and a -number of clusters, and receive the cluster assignments in return. Note that -the dataset must be column-major -- that is, one column corresponds to one -point. See \ref matrices "the matrices guide" for more information. - -@code -#include - -using namespace mlpack::kmeans; - -// The dataset we are clustering. -extern arma::mat data; -// The number of clusters we are getting. -extern size_t clusters; - -// The assignments will be stored in this vector. -arma::Row assignments; - -// Initialize with the default arguments. -KMeans<> k; -k.Cluster(data, clusters, assignments); -@endcode - -Now, the vector \c assignments holds the cluster assignments of each point in -the dataset. - -@subsection kmeans_ex2_kmtut Running k-means and getting centroids of clusters - -Often it is useful to not only have the cluster assignments, but the centroids -of each cluster. Another overload of \c Cluster() makes this easily possible: - -@code -#include - -using namespace mlpack::kmeans; - -// The dataset we are clustering. -extern arma::mat data; -// The number of clusters we are getting. -extern size_t clusters; - -// The assignments will be stored in this vector. -arma::Row assignments; -// The centroids will be stored in this matrix. -arma::mat centroids; - -// Initialize with the default arguments. -KMeans<> k; -k.Cluster(data, clusters, assignments, centroids); -@endcode - -Note that the centroids matrix has columns equal to the number of clusters and -rows equal to the dimensionality of the dataset. Each column represents the -centroid of the according cluster -- \c centroids.col(0) represents the -centroid of the first cluster. - -@subsection kmeans_ex3_kmtut Limiting the maximum number of iterations - -The first argument to the constructor allows specification of the maximum number -of iterations. This is useful because often, the k-means algorithm does not -converge, and is terminated after a number of iterations. Setting this -parameter to 0 indicates that the algorithm will run until convergence -- note -that in some cases, convergence may never happen. The default maximum number of -iterations is 1000. - -@code -// The first argument is the maximum number of iterations. Here we set it to -// 500 iterations. -KMeans<> k(500); -@endcode - -Then you can run \c Cluster() as normal. - -@subsection kmeans_ex5_kmtut Setting initial cluster assignments - -If you have an initial guess for the cluster assignments for each point, you can -fill the assignments vector with the guess and then pass an extra boolean -(initialAssignmentGuess) as true to the \c Cluster() method. Below are examples -for either overload of \c Cluster(). - -@code -#include - -using namespace mlpack::kmeans; - -// The dataset we are clustering on. -extern arma::mat dataset; -// The number of clusters we are obtaining. -extern size_t clusters; - -// A vector pre-filled with initial assignment guesses. -extern arma::Row assignments; - -KMeans<> k; - -// The boolean set to true indicates that our assignments vector is filled with -// initial guesses. -k.Cluster(dataset, clusters, assignments, true); -@endcode - -@code -#include - -using namespace mlpack::kmeans; - -// The dataset we are clustering on. -extern arma::mat dataset; -// The number of clusters we are obtaining. -extern size_t clusters; - -// A vector pre-filled with initial assignment guesses. -extern arma::Row assignments; - -// This will hold the centroids of the finished clusters. -arma::mat centroids; - -KMeans<> k; - -// The boolean set to true indicates that our assignments vector is filled with -// initial guesses. -k.Cluster(dataset, clusters, assignments, centroids, true); -@endcode - -@note -If you have a heuristic or algorithm which makes initial guesses, a -more elegant solution is to create a new class fulfilling the -InitialPartitionPolicy template policy. See \ref kmeans_initial_partition_kmtut -"the section about changing the initial partitioning strategy" for more details. - -@par - -@note -If you set the InitialPartitionPolicy parameter to something other than the -default but give an initial cluster assignment guess, the InitialPartitionPolicy -will not be used to initialize the algorithm. See \ref kmeans_initial_partition_kmtut -"the section about changing the initial partitioning strategy" -for more details. - -@subsection kmeans_ex6_kmtut Setting initial cluster centroids - -An equally important option to being able to make initial cluster assignment -guesses is to make initial cluster centroid guesses without having to assign -each point in the dataset to an initial cluster. This is similar to the -previous section, but now you must pass two extra booleans -- the first -(initialAssignmentGuess) as false, indicating that there are not initial cluster -assignment guesses, and the second (initialCentroidGuess) as true, indicating -that the centroids matrix is filled with initial centroid guesses. - -This, of course, only works with the overload of \c Cluster() that takes a -matrix to put the resulting centroids in. Below is an example. - -@code -#include - -using namespace mlpack::kmeans; - -// The dataset we are clustering on. -extern arma::mat dataset; -// The number of clusters we are obtaining. -extern size_t clusters; - -// A matrix pre-filled with guesses for the initial cluster centroids. -extern arma::mat centroids; - -// This will be filled with the final cluster assignments for each point. -arma::Row assignments; - -KMeans<> k; - -// Remember, the first boolean indicates that we are not giving initial -// assignment guesses, and the second boolean indicates that we are giving -// initial centroid guesses. -k.Cluster(dataset, clusters, assignments, centroids, false, true); -@endcode - -@note -If you have a heuristic or algorithm which makes initial guesses, a -more elegant solution is to create a new class fulfilling the -InitialPartitionPolicy template policy. See \ref kmeans_initial_partition_kmtut -"the section about changing the initial partitioning strategy" for more details. - -@par - -@note -If you set the InitialPartitionPolicy parameter to something other than the -default but give an initial cluster centroid guess, the InitialPartitionPolicy -will not be used to initialize the algorithm. See \ref kmeans_initial_partition_kmtut -"the section about changing the initial partitioning strategy" for more details. - -@subsection kmeans_ex7_kmtut Running sparse k-means - -The \c Cluster() function can work on both sparse and dense matrices, so all of -the above examples can be used with sparse matrices instead, if the fifth -template parameter is modified. Below is a simple example. Note that the -centroids are returned as a dense matrix, because the centroids of collections -of sparse points are not generally sparse. - -@code -// The sparse dataset. -extern arma::sp_mat sparseDataset; -// The number of clusters. -extern size_t clusters; - -// The assignments will be stored in this vector. -arma::Row assignments; -// The centroids of each cluster will be stored in this sparse matrix. -arma::sp_mat sparseCentroids; - -// We must change the fifth (and last) template parameter. -KMeans k; -k.Cluster(sparseDataset, clusters, assignments, sparseCentroids); -@endcode - -@section kmeans_template_kmtut Template parameters for the 'KMeans' class - -The \c KMeans<> class also takes three template parameters, which can be -modified to change the behavior of the k-means algorithm. There are three -template parameters: - - - \c MetricType: controls the distance metric used for clustering (by - default, the squared Euclidean distance is used) - - \c InitialPartitionPolicy: the method by which initial clusters are set; by - default, \ref mlpack::kmeans::SampleInitialization "SampleInitialization" is - used - - \c EmptyClusterPolicy: the action taken when an empty cluster is encountered; - by default, \ref mlpack::kmeans::MaxVarianceNewCluster "MaxVarianceNewCluster" - is used - - \c LloydStepType: this defines the strategy used to make a single Lloyd - iteration; by default this is the typical Lloyd iteration specified in - \ref mlpack::kmeans::NaiveKMeans "NaiveKMeans" - - \c MatType: type of data matrix to use for clustering - -The class is defined like below: - -@code -template< - typename DistanceMetric = mlpack::metric::SquaredEuclideanDistance, - typename InitialPartitionPolicy = SampleInitialization, - typename EmptyClusterPolicy = MaxVarianceNewCluster, - template class LloydStepType = NaiveKMeans, - typename MatType = arma::mat -> -class KMeans; -@endcode - -In the following sections, each policy is described further, with examples of -how to modify them. - -@subsection kmeans_metric_kmtut Changing the distance metric used for k-means - -Most machine learning algorithms in \b mlpack support modifying the distance -metric, and \c KMeans<> is no exception. Similar to \ref -mlpack::neighbor::NeighborSearch "NeighborSearch" (see \ref -metric_type_doc_nstut "the section in the NeighborSearch tutorial"), any class -in mlpack::metric can be given as an argument. The mlpack::metric::LMetric -class is a good example implementation. - -A class fulfilling the MetricType policy must provide the following two -functions: - -@code -// Empty constructor is required. -MetricType(); - -// Computer the distance between two points. -template -double Evaluate(const VecType& a, const VecType& b); -@endcode - -Most of the standard metrics that could be used are stateless and therefore the -\c Evaluate() method is implemented statically. However, there are metrics, -such as the Mahalanobis distance (mlpack::metric::MahalanobisDistance), that -store state. To this end, an instantiated MetricType object is stored within the -\c KMeans class. The example below shows how to pass an instantiated -MahalanobisDistance in the constructor. - -@code -// The initialized Mahalanobis distance. -extern mlpack::metric::MahalanobisDistance distance; - -// We keep the default arguments for the maximum number of iterations, but pass -// our instantiated metric. -KMeans k(1000, distance); -@endcode - -@note -While the MetricType policy only requires two methods, one of which is an empty -constructor, more can always be added. mlpack::metric::MahalanobisDistance also -has constructors with parameters, because it is a stateful metric. - -@subsection kmeans_initial_partition_kmtut Changing the initial partitioning strategy used for k-means - -There have been many initial cluster strategies for k-means proposed in the -literature. Fortunately, the \c KMeans<> class makes it very easy to implement -one of these methods and plug it in without needing to modify the existing -algorithm code at all. - -By default, the \c KMeans<> class uses mlpack::kmeans::SampleInitialization, -which randomly samples points as initial centroids. However, writing a new -policy is simple; it needs to only implement the following functions: - -@code -// Empty constructor is required. -InitialPartitionPolicy(); - -// Only *one* of the following two functions is required! You should implement -// whichever you find more convenient to implement. - -// This function is called to initialize the clusters and returns centroids. -template -void Cluster(MatType& data, - const size_t clusters, - arma::mat& centroids); - -// This function is called to initialize the clusters and returns individual -// point assignments. The centroids will then be calculated from the given -// assignments. -template -void Cluster(MatType& data, - const size_t clusters, - arma::Row assignments); -@endcode - -The templatization of the \c Cluster() function allows both dense and sparse -matrices to be passed in. If the desired policy does not work with sparse (or -dense) matrices, then the method can be written specifically for one type of -matrix -- however, be warned that if you try to use \c KMeans with that policy -and the wrong type of matrix, you will get many ugly compilation errors! - -@code -// The Cluster() function specialized for dense matrices. -void Cluster(arma::mat& data, - const size_t clusters, - arma::Row assignments); -@endcode - -Note that only one of the two possible \c Cluster() functions are required. -This is because sometimes it is easier to express an initial partitioning policy -as something that returns point assignments, and sometimes it is easier to -express the policy as something that returns centroids. The KMeans<> class will -use whichever of these two functions is given; if both are given, the overload -that returns centroids will be preferred. - -One alternate to the default SampleInitialization policy is the RefinedStart -policy, which is an implementation of the Bradley and Fayyad approach for -finding initial points detailed in "Refined initial points for k-means -clustering" and other places in this document. Another option is the -RandomPartition class, which randomly assigns points to clusters, but this may -not work very well for most settings. See the documentation for -mlpack::kmeans::RefinedStart and mlpack::kmeans::RandomPartition for more -information. - -If the \c Cluster() method returns point assignments instead of centroids, then -valid initial assignments must be returned for every point in the dataset. - -As with the MetricType template parameter, an initialized InitialPartitionPolicy -can be passed to the constructor of \c KMeans as a fourth argument. - -@subsection kmeans_empty_cluster_kmtut Changing the action taken when an empty cluster is encountered - -Sometimes, during clustering, a situation will arise where a cluster has no -points in it. The \c KMeans class allows easy customization of the action to be -taken when this occurs. By default, the point furthest from the centroid of the -cluster with maximum variance is taken as the centroid of the empty cluster; -this is implemented in the mlpack::kmeans::MaxVarianceNewCluster class. Another -alternate choice is the mlpack::kmeans::AllowEmptyClusters class, which simply -allows empty clusters to persist. - -A custom policy can be written and it must implement the following methods: - -@code -// Empty constructor is required. -EmptyClusterPolicy(); - -// This function is called when an empty cluster is encountered. emptyCluster -// indicates the cluster which is empty, and then the clusterCounts and -// assignments are meant to be modified by the function. The function should -// return the number of modified points. -template -size_t EmptyCluster(const MatType& data, - const size_t emptyCluster, - const MatType& centroids, - arma::Col& clusterCounts, - arma::Row& assignments); -@endcode - -The \c EmptyCluster() function is called for each cluster that is empty at each -iteration of the algorithm. As with InitialPartitionPolicy, the \c -EmptyCluster() function does not need to be generalized to support both dense -and sparse matrices -- but usage with the wrong type of matrix will cause -compilation errors. - -Like the other template parameters to \c KMeans, EmptyClusterPolicy -implementations that have state can be passed to the constructor of \c KMeans as -a fifth argument. See the kmeans::KMeans documentation for further details. - -@subsection kmeans_lloyd_kmtut The LloydStepType template parameter - -The internal algorithm used for a single step of the k-means algorithm can -easily be changed; \b mlpack implements several existing classes that satisfy -the \c LloydStepType policy: - - - mlpack::kmeans::NaiveKMeans - - mlpack::kmeans::ElkanKMeans - - mlpack::kmeans::HamerlyKMeans - - mlpack::kmeans::PellegMooreKMeans - - mlpack::kmeans::DualTreeKMeans - -Note that the \c LloydStepType policy is itself a template template parameter, -and must accept two template parameters of its own: - - - \c MetricType: the type of metric to use - - \c MatType: the type of data matrix to use - -The \c LloydStepType policy also mandates three functions: - - - a constructor: LloydStepType(const MatType& dataset, MetricType& - metric); - - an \c Iterate() function: - -@code -/** - * Run a single iteration of the Lloyd algorithm, updating the given centroids - * into the newCentroids matrix. If any cluster is empty (that is, if any - * cluster has no points assigned to it), then the centroid associated with - * that cluster may be filled with invalid data (it will be corrected later). - * - * @param centroids Current cluster centroids. - * @param newCentroids New cluster centroids. - * @param counts Number of points in each cluster at the end of the iteration. - */ -double Iterate(const arma::mat& centroids, - arma::mat& newCentroids, - arma::Col& counts); -@endcode - - - a function to get the number of distance calculations: - -@code -size_t DistanceCalculations() const { return distanceCalculations; } -@endcode - -Note that \c Iterate() does not need to return valid centroids if the cluster is -empty. This is because \c EmptyClusterPolicy will handle the empty centroid. -This behavior can be used to avoid small amounts of computation. - -For examples, see the five aforementioned implementations of classes that -satisfy the \c LloydStepType policy. - -@section further_doc_kmtut Further documentation - -For further documentation on the KMeans class, consult the \ref -mlpack::kmeans::KMeans "complete API documentation". - -*/ diff --git a/doc/tutorials/linear_regression/linear_regression.txt b/doc/tutorials/linear_regression.md similarity index 64% rename from doc/tutorials/linear_regression/linear_regression.txt rename to doc/tutorials/linear_regression.md index 751ff0c935..1ab16156f4 100644 --- a/doc/tutorials/linear_regression/linear_regression.txt +++ b/doc/tutorials/linear_regression.md @@ -1,83 +1,52 @@ -/*! - -@file linear_regression.txt -@author James Cline -@brief Tutorial for how to use the LinearRegression class. - -@page lrtutorial Linear/ridge regression tutorial (mlpack_linear_regression) - -@section intro_lrtut Introduction +# Linear/ridge regression tutorial Linear regression and ridge regression are simple machine learning techniques -that aim to estimate the parameters of a linear model. Assuming we have \f$n\f$ -\b predictor points \f$\mathbf{x_i}, 0 \le i < n\f$ of dimensionality \f$d\f$ -and \f$n\f$ responses \f$y_i, 0 \le i < n\f$, we are trying to estimate the best -fit for \f$\beta_i, 0 \le i \le d\f$ in the linear model +that aim to estimate the parameters of a linear model. Assuming we have `n` +*predictor* points `x_i`, of dimensionality `d`, and `n` responses `y_i`, we are +trying to estimate the best fit for `b_i` with `0 <= i <= d` in the linear model -\f[ -y_i = \beta_0 + \displaystyle\sum_{j = 1}^{d} \beta_j x_{ij} -\f] +``` +y_i = b_0 + sum_j (b_j x_ij) +``` -for each predictor \f$\mathbf{x_i}\f$ and response \f$y_i\f$. If we take each -predictor \f$\mathbf{x_i}\f$ as a row in the matrix \f$\mathbf{X}\f$ and each -response \f$y_i\f$ as an entry of the vector \f$\mathbf{y}\f$, we can represent -the model in vector form: +for each predictor `x_i` and response `y_i`. If we take each predictor `x_i` as +a row in the matrix `X` and each response `y_i` as an entry of the vector `y`, +we can represent the model in vector form: -\f[ -\mathbf{y} = \mathbf{X} \mathbf{\beta} + \beta_0 -\f] +``` +y = Xb + b_0 +``` -The result of this method is the vector \f$\mathbf{\beta}\f$, including the -offset term (or intercept term) \f$\beta_0\f$. +The result of this method is the vector `b`, including the offset term (or +intercept term) `b_0`. -\b mlpack provides: +## Command-line `mlpack_linear_regression` - - a \ref cli_lrtut "simple command-line executable" to perform linear regression or ridge regression - - a \ref linreg_lrtut "simple C++ interface" to perform linear regression or ridge regression - -@section toc_lrtut Table of Contents - -A list of all the sections this tutorial contains. - - - \ref intro_lrtut - - \ref toc_lrtut - - \ref cli_lrtut - - \ref cli_ex1_lrtut - - \ref cli_ex2_lrtut - - \ref cli_ex3_lrtut - - \ref cli_ex4_lrtut - - \ref cli_ex5_lrtut - - \ref linreg_lrtut - - \ref linreg_ex1_lrtut - - \ref linreg_ex2_lrtut - - \ref linreg_ex3_lrtut - - \ref linreg_ex4_lrtut - - \ref linreg_ex5_lrtut - - \ref further_doc_lrtut - -@section cli_lrtut Command-Line 'mlpack_linear_regression' - -The simplest way to perform linear regression or ridge regression in \b mlpack -is to use the \c mlpack_linear_regression executable. This program will perform -linear regression and place the resultant coefficients into one file. +The simplest way to perform linear regression or ridge regression in mlpack +is to use the `mlpack_linear_regression` program. This program will perform +linear regression and place the resultant coefficients into one file. Note that +this guide details the `mlpack_linear_regression` command-line program, but +because mlpack also has bindings to other languages, functions like +`linear_regression()` exist in Python and Julia, and each example below can be +easily adapted to those languages. The output file holds a vector of coefficients in increasing order of dimension; -that is, the offset term (\f$\beta_0\f$), the coefficient for dimension 1 -(\f$\beta_1\f$, then dimension 2 (\f$\beta_2\f$) and so forth, as well as the -intercept. This executable can also predict the \f$y\f$ values of a second -dataset based on the computed coefficients. +that is, the offset term (`b_0`), the coefficient for dimension 1 (`b_1`, then +dimension 2 (`b_2`) and so forth, as well as the intercept. This executable can +also predict the `y` values of a second dataset based on the computed +coefficients. -Below are several examples of simple usage (and the resultant output). The -\c option is used so that verbose output is given. Further documentation on -each individual option can be found by typing +Below are several examples of simple usage (and the resultant output). The `-v` +option is used so that verbose output is given. Further documentation on each +individual option can be found by typing -@code +```sh $ mlpack_linear_regression --help -@endcode +``` -@subsection cli_ex1_lrtut One file, generating the function coefficients +### One file, generating the function coefficients -@code +```sh $ mlpack_linear_regression --training_file dataset.csv -v -M lr.xml [INFO ] Loading 'dataset.csv' as CSV data. Size is 2 x 5. [INFO ] @@ -99,13 +68,13 @@ $ mlpack_linear_regression --training_file dataset.csv -v -M lr.xml [INFO ] loading_data: 0.000220s [INFO ] regression: 0.000392s [INFO ] total_time: 0.001920s -@endcode +``` Convenient program timers are given for different parts of the calculation at the bottom of the output, as well as the parameters the simulation was run with. -Now, if we look at the output model file, which is \c lr.xml, +Now, if we look at the output model file, which is `lr.xml`, -@code +```sh $ cat dataset.csv 0,0 1,1 @@ -129,39 +98,37 @@ $ cat lr.xml true +``` - -@endcode - -As you can see, the function for this input is \f$f(y)=0+1x_1\f$. We can see -that the model we have trained catches this; in the \c \ section of -\c lr.xml, we can see that there are two elements, which are (approximately) 0 +As you can see, the function for this input is `f(y) = 0 + 1 x_1`. We can see +that the model we have trained catches this; in the `` section of +`lr.xml`, we can see that there are two elements, which are (approximately) 0 and 1. The first element corresponds to the intercept 0, and the second column -corresponds to the coefficient 1 for the variable \f$x_1\f$. Note that in this +corresponds to the coefficient 1 for the variable `x_1`. Note that in this example, the regressors for the dataset are the second column. That is, the -dataset is one dimensional, and the last column has the \f$y\f$ values, or +dataset is one dimensional, and the last column has the `y` values, or responses, for each row. You can specify these responses in a separate file if -you want, using the \c --input_responses, or \c -r, option. +you want, using the `--input_responses`, or `-r`, option. -@subsection cli_ex2_lrtut Train a multivariate linear regression model +### Train a multivariate linear regression model Multivariate linear regression means that the response variable is predicted by more than just one input variable. In this example we will try to fit a -multivariate linear regression model to data that contains four variables, stored in -\c dataset_2.csv. +multivariate linear regression model to data that contains four variables, +stored in `dataset_2.csv`. -@code +```sh $ cat dataset_2.csv 0,0,0,0,14 1,1,1,1,24 2,1,0,2,27 1,2,2,2,32 -1,-3,0,2,17 -@endcode +``` -Now let's run \c mlpack_linear_regression as usual: +Now let's run `mlpack_linear_regression` as usual: -@code +```sh $ mlpack_linear_regression --training_file dataset_2.csv -v -M lr.xml [INFO ] Loading 'dataset_2.csv' as CSV data. Size is 5 x 5. [INFO ] @@ -183,7 +150,7 @@ $ mlpack_linear_regression --training_file dataset_2.csv -v -M lr.xml [INFO ] regression: 0.000049s [INFO ] total_time: 0.000118s -$ cat lr.xml +$ cat lr.xml @@ -202,15 +169,16 @@ $ cat lr.xml true -@endcode +``` -If we take a look at the \c lr.xml output we can see the \c \ part has five elements which -the first corresponds to \f$\beta_0\f$ , the second corresponds to \f$\beta_1\f$ , and so on. This is equivalent -to \f$f(y) = \beta_0 + \beta_1x_1 + \beta_2x_2 + \beta_3x_3 + \beta_4x_4\f$ or \f$f(y)=14+2x_1+1x_2+3x_3+4x_4\f$. +If we take a look at the `lr.xml` output we can see the `` part has +five elements which the first corresponds to `b_0`, the second corresponds to +`b_1` , and so on. This is equivalent to `f(y) = b_0 + b_1 x_1 + b_2 x_2 + b_3 +x_3 + b_4 x_4`, or `f(y) = 14 + 2 x_1 + 1 x_2 + 3 x_3 + 4 x_4`. -@subsection cli_ex3_lrtut Compute model and predict at the same time +### Compute model and predict at the same time -@code +```sh $ mlpack_linear_regression --training_file dataset.csv --test_file predict.csv --output_predictions_file predictions.csv \ > -v [WARN ] '--output_predictions_file (-o)' ignored because '--test_file (-T)' is specified! @@ -256,16 +224,16 @@ $ cat predictions.csv 2.0000000000e+00 3.0000000000e+00 4.0000000000e+00 -@endcode +``` We used the same dataset, so we got the same parameters. The key thing to note -about the \c predict.csv dataset is that it has the same dimensionality as the -dataset used to create the model, one. If the model generating dataset has -\f$d\f$ dimensions, so must the dataset we want to predict for. +about the `predict.csv` dataset is that it has the same dimensionality as the +dataset used to create the model, one. If the model generating dataset has `d` +dimensions, so must the dataset we want to predict for. -@subsection cli_ex4_lrtut Prediction using a precomputed model +### Prediction using a precomputed model -@code +```sh $ mlpack_linear_regression --input_model_file lr.xml --test_file predict.csv --output_predictions_file predictions.csv -v [WARN ] '--output_predictions_file (-o)' ignored because '--test_file (-T)' is specified! [INFO ] Loading 'predict.csv' as raw ASCII formatted data. Size is 1 x 3. @@ -319,32 +287,21 @@ $ cat predictions.csv 2.0000000000e+00 3.0000000000e+00 4.0000000000e+00 -@endcode +``` -@subsection cli_ex5_lrtut Using ridge regression +### Using ridge regression Sometimes, the input matrix of predictors has a covariance matrix that is not invertible, or the system is overdetermined. In this case, ridge regression is useful: it adds a normalization term to the covariance matrix to make it invertible. Ridge regression is a standard technique and documentation for the mathematics behind it can be found anywhere on the Internet. In short, the -covariance matrix +covariance matrix `X' X` is replaced with `X' X + l I` where `I` is the identity +matrix. So, an `l` parameter greater than zero should be specified to perform +ridge regression, using the `--lambda` (or `-l`) option. An example is given +below. -\f[ -\mathbf{X}' \mathbf{X} -\f] - -is replaced with - -\f[ -\mathbf{X}' \mathbf{X} + \lambda \mathbf{I} -\f] - -where \f$\mathbf{I}\f$ is the identity matrix. So, a \f$\lambda\f$ parameter -greater than zero should be specified to perform ridge regression, using the -\c --lambda (or \c -l) option. An example is given below. - -@code +```sh $ mlpack_linear_regression --training_file dataset.csv -v --lambda 0.5 -M lr.xml [INFO ] Loading 'dataset.csv' as CSV data. Size is 2 x 5. [INFO ] @@ -366,33 +323,34 @@ $ mlpack_linear_regression --training_file dataset.csv -v --lambda 0.5 -M lr.xml [INFO ] loading_data: 0.000170s [INFO ] regression: 0.000332s [INFO ] total_time: 0.001835s -@endcode +``` -Further documentation on options should be found by using the \c --help option. +Further documentation on options should be found by using the `--help` option. -@section linreg_lrtut The 'LinearRegression' class +## The `LinearRegression` class -The 'LinearRegression' class is a simple implementation of linear regression. +The `LinearRegression` class is a simple implementation of linear regression. -Using the LinearRegression class is very simple. It has two available +Using the `LinearRegression` class is very simple. It has two available constructors; one for generating a model from a matrix of predictors and a vector of responses, and one for loading an already computed model from a given file. The class provides one method that performs computation: -@code + +```c++ void Predict(const arma::mat& points, arma::vec& predictions); -@endcode +``` Once you have generated or loaded a model, you can call this method and pass it a matrix of data points to predict values for using the model. The second parameter, predictions, will be modified to contain the predicted values corresponding to each row of the points matrix. -@subsection linreg_ex1_lrtut Generating a model +### Generating a model -@code -#include +```c++ +#include using namespace mlpack::regression; @@ -404,38 +362,38 @@ LinearRegression lr(data, responses); // Get the parameters, or coefficients. arma::vec parameters = lr.Parameters(); -@endcode +``` -@subsection linreg_ex2_lrtut Setting a model +### Setting a model Assuming you already have a model and do not need to create one, this is how -you would set the parameters for a LinearRegression instance. +you would set the parameters for a `LinearRegression` instance. -@code +```c++ arma::vec parameters; // Your model. LinearRegression lr; // Create a new LinearRegression instance or reuse one. lr.Parameters() = parameters; // Set the model. -@endcode +``` -@subsection linreg_ex3_lrtut Load a model from a file +### Load a model from file If you have a generated model in a file somewhere you would like to load and -use, you can use \c data::Load() to load it. +use, you can use `data::Load()` to load it. -@code +```c++ std::string filename; // The path and name of your file. LinearRegression lr; data::Load(filename, "lr_model", lr); -@endcode +``` -@subsection linreg_ex4_lrtut Prediction +### Prediction Once you have generated or loaded a model using one of the aforementioned methods, you can predict values for a dataset. -@code +```c++ LinearRegression lr(); // Load or generate your model. @@ -447,16 +405,16 @@ arma::vec predictions; lr.Predict(points, predictions); // Predict. // Now, the vector 'predictions' will contain the predicted values. -@endcode +``` -@subsection linreg_ex5_lrtut Setting lambda for ridge regression +### Setting lambda for ridge regression -As discussed in \ref cli_ex4_lrtut, ridge regression is useful when the +As discussed in an earlier example, ridge regression is useful when the covariance of the predictors is not invertible. The standard constructor can be used to set a value of lambda: -@code -#include +```c++ +#include using namespace mlpack::regression; @@ -468,20 +426,19 @@ LinearRegression lr(data, responses, 0.5); // Get the parameters, or coefficients. arma::vec parameters = lr.Parameters(); -@endcode +``` -In addition, the \c Lambda() function can be used to get or modify the lambda +In addition, the `Lambda()` function can be used to get or modify the lambda value: -@code +```c++ LinearRegression lr; lr.Lambda() = 0.5; Log::Info << "Lambda is " << lr.Lambda() << "." << std::endl; -@endcode +``` -@section further_doc_lrtut Further documentation +## Further documentation -For further documentation on the LinearRegression class, consult the -\ref mlpack::regression::LinearRegression "complete API documentation". - -*/ +For further documentation on the LinearRegression class, consult the comments in +the source code of the `LinearRegression` class, found in +`mlpack/methods/linear_regression/`. diff --git a/doc/tutorials/neighbor_search/neighbor_search.txt b/doc/tutorials/neighbor_search.md similarity index 62% rename from doc/tutorials/neighbor_search/neighbor_search.txt rename to doc/tutorials/neighbor_search.md index 6710580cd6..fb1baa6a7b 100644 --- a/doc/tutorials/neighbor_search/neighbor_search.txt +++ b/doc/tutorials/neighbor_search.md @@ -1,72 +1,48 @@ -/*! - -@file neighbor_search.txt -@author Ryan Curtin -@brief Tutorial for how to use the NeighborSearch class. - -@page nstutorial NeighborSearch tutorial (k-nearest-neighbors) - -@section intro_nstut Introduction +# NeighborSearch tutorial (k-nearest-neighbors) Nearest-neighbors search is a common machine learning task. In this setting, we -have a \b query and a \b reference dataset. For each point in the \b query -dataset, we wish to know the \f$k\f$ points in the \b reference dataset which -are closest to the given query point. +have a *query* and a *reference* dataset. For each point in the *query* +dataset, we wish to know the `k` points in the *reference* dataset which are +closest to the given query point. Alternately, if the query and reference datasets are the same, the problem can -be stated more simply: for each point in the dataset, we wish to know the -\f$k\f$ nearest points to that point. +be stated more simply: for each point in the dataset, we wish to know the `k` +nearest points to that point. -\b mlpack provides: +mlpack provides: - - a \ref cli_nstut "simple command-line executable" to run nearest-neighbors search - (and furthest-neighbors search) - - a \ref knn_nstut "simple C++ interface" to perform nearest-neighbors search (and + - a simple command-line executable to run nearest-neighbors search (and furthest-neighbors search) - - a \ref neighborsearch_nstut "generic, extensible, and powerful C++ class (NeighborSearch)" for complex usage + - a simple C++ interface to perform nearest-neighbors search (and + furthest-neighbors search) + - a generic, extensible, and powerful C++ class (`NeighborSearch`) for complex + usage -@section toc_nstut Table of Contents +## Command-line `mlpack_knn` -A list of all the sections this tutorial contains. +The simplest way to perform nearest-neighbors search in mlpack is to use the +`mlpack_knn` executable. *(Note that mlpack also provides bindings to other +languages, so, e.g., the `knn()` function is available in Python and Julia and +has the same options. So, any example here can be readily adapted to another +language that mlpack provides bindings for.)* - - \ref intro_nstut - - \ref toc_nstut - - \ref cli_nstut - - \ref cli_ex1_nstut - - \ref cli_ex2_nstut - - \ref cli_ex3_nstut - - \ref knn_nstut - - \ref knn_ex1_nstut - - \ref knn_ex2_nstut - - \ref knn_ex3_nstut - - \ref neighborsearch_nstut - - \ref sort_policy_doc_nstut - - \ref metric_type_doc_nstut - - \ref mat_type_doc_nstut - - \ref tree_type_doc_nstut - - \ref traverser_type_doc_nstut - - \ref further_doc_nstut +The `mlpack_knn` program will perform nearest-neighbors search and place the +resultant neighbors into one file and the resultant distances into another. The +output files are organized such that the first row corresponds to the nearest +neighbors of the first query point, with the first column corresponding to the +nearest neighbor, and so forth. -@section cli_nstut Command-Line 'mlpack_knn' - -The simplest way to perform nearest-neighbors search in \b mlpack is to use the -\c mlpack_knn executable. This program will perform nearest-neighbors search -and place the resultant neighbors into one file and the resultant distances into -another. The output files are organized such that the first row corresponds to -the nearest neighbors of the first query point, with the first column -corresponding to the nearest neighbor, and so forth. - -Below are several examples of simple usage (and the resultant output). The -\c -v option is used so that output is given. Further documentation on each +Below are several examples of simple usage (and the resultant output). The `-v` +option is used so that output is given. Further documentation on each individual option can be found by typing -@code +```sh $ mlpack_knn --help -@endcode +``` -@subsection cli_ex1_nstut One dataset, 5 nearest neighbors +### One dataset, 5 nearest neighbors -@code +```sh $ mlpack_knn -r dataset.csv -n neighbors_out.csv -d distances_out.csv -k 5 -v [INFO ] Loading 'dataset.csv' as CSV data. Size is 3 x 1000. [INFO ] Loaded reference data from 'dataset.csv' (3 x 1000). @@ -104,13 +80,13 @@ $ mlpack_knn -r dataset.csv -n neighbors_out.csv -d distances_out.csv -k 5 -v [INFO ] saving_data: 0.003843s [INFO ] total_time: 0.126036s [INFO ] tree_building: 0.003442s -@endcode +``` Convenient program timers are given for different parts of the calculation at the bottom of the output, as well as the parameters the simulation was run with. Now, if we look at the output files: -@code +```sh $ head neighbors_out.csv 862,344,224,43,885 703,499,805,639,450 @@ -134,16 +110,16 @@ $ head distances_out.csv 7.005321598247e-02,9.131417221561e-02,9.498248889074e-02,9.897964162308e-02,1.121202216165e-01 5.295654132754e-02,5.509877761894e-02,8.108227366619e-02,9.785461174861e-02,1.043968140367e-01 3.992859920333e-02,4.471418646159e-02,7.346053904990e-02,9.181982339584e-02,9.843075910782e-02 -@endcode +``` So, the nearest neighbor to point 0 is point 862, with a distance of -5.986076164057e-02. The second nearest neighbor to point 0 is point 344, with a -distance of 7.664920518084e-02. The third nearest neighbor to point 5 is point -751, with a distance of 1.085637706630e-01. +`5.986076164057e-02`. The second nearest neighbor to point 0 is point 344, with +a distance of `7.664920518084e-02`. The third nearest neighbor to point 5 is +point 751, with a distance of `1.085637706630e-01`. -@subsection cli_ex2_nstut Query and reference dataset, 10 nearest neighbors +### Query and reference dataset, 10 nearest neighbors -@code +```sh $ mlpack_knn -q query_dataset.csv -r reference_dataset.csv \ > -n neighbors_out.csv -d distances_out.csv -k 10 -v [INFO ] Loading 'reference_dataset.csv' as CSV data. Size is 3 x 1000. @@ -184,11 +160,11 @@ $ mlpack_knn -q query_dataset.csv -r reference_dataset.csv \ [INFO ] saving_data: 0.000755s [INFO ] total_time: 0.032197s [INFO ] tree_building: 0.002590s -@endcode +``` -@subsection cli_ex3_nstut One dataset, 3 nearest neighbors, leaf size of 15 points +### One dataset, 3 nearest neighbors, leaf size of 15 points -@code +```sh $ mlpack_knn -r dataset.csv -n neighbors_out.csv -d distances_out.csv -k 3 -l 15 -v [INFO ] Loading 'dataset.csv' as CSV data. Size is 3 x 1000. [INFO ] Loaded reference data from 'dataset.csv' (3 x 1000). @@ -226,33 +202,33 @@ $ mlpack_knn -r dataset.csv -n neighbors_out.csv -d distances_out.csv -k 3 -l 15 [INFO ] saving_data: 0.002369s [INFO ] total_time: 0.069277s [INFO ] tree_building: 0.002713s -@endcode +``` -Further documentation on options should be found by using the --help option. +Further documentation on options should be found by using the `--help` option. -@section knn_nstut The 'KNN' class +## The `KNN` class -The 'KNN' class is, specifically, a typedef of the more extensible -NeighborSearch class, querying for nearest neighbors using the Euclidean +The `KNN` class is, specifically, a typedef of the more extensible +`NeighborSearch` class, querying for nearest neighbors using the Euclidean distance. -@code +```c++ typedef NeighborSearch KNN; -@endcode +``` -Using the KNN class is particularly simple; first, the object must be +Using the `KNN` class is particularly simple; first, the object must be constructed and given a dataset. Then, the method is run, and two matrices are returned: one which holds the indices of the nearest neighbors, and one which holds the distances of the nearest neighbors. These are of the same structure -as the output --neighbors_file and --distances_file for the CLI interface (see -above). A handful of examples of simple usage of the KNN class are given -below. +as the output `--neighbors_file` and `--distances_file` for the command-line +program (see above). A handful of examples of simple usage of the KNN class are +given below. -@subsection knn_ex1_nstut 5 nearest neighbors on a single dataset +### 5 nearest neighbors on a single dataset -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -266,14 +242,15 @@ arma::Mat resultingNeighbors; arma::mat resultingDistances; a.Search(5, resultingNeighbors, resultingDistances); -@endcode +``` -The output of the search is stored in resultingNeighbors and resultingDistances. +The output of the search is stored in `resultingNeighbors` and +`resultingDistances`. -@subsection knn_ex2_nstut 10 nearest neighbors on a query and reference dataset +### 10 nearest neighbors on a query and reference dataset -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -287,14 +264,14 @@ arma::Mat resultingNeighbors; arma::mat resultingDistances; a.Search(queryData, 10, resultingNeighbors, resultingDistances); -@endcode +``` -@subsection knn_ex3_nstut Naive (exhaustive) search for 6 nearest neighbors on one dataset +### Naive (exhaustive) search for 6 nearest neighbors on one dataset -This example uses the O(n^2) naive search (not the tree-based search). +This example uses the `O(n^2)` naive search (not the tree-based search). -@code -#include +```c++ +#include using namespace mlpack::neighbor; @@ -308,16 +285,16 @@ arma::Mat resultingNeighbors; arma::mat resultingDistances; a.Search(6, resultingNeighbors, resultingDistances); -@endcode +``` Needless to say, naive search can be very slow... -@section neighborsearch_nstut The extensible 'NeighborSearch' class +## The extensible `NeighborSearch` class -The NeighborSearch class is very extensible, having the following template +The `NeighborSearch` class is very extensible, having the following template arguments: -@code +```c++ template< typename SortPolicy = NearestNeighborSort, typename MetricType = mlpack::metric::EuclideanDistance, @@ -330,21 +307,21 @@ template< MatType>::template DualTreeTraverser> > class NeighborSearch; -@endcode +``` By choosing different components for each of these template classes, a very arbitrary neighbor searching object can be constructed. Note that each of these template parameters have defaults, so it is not necessary to specify each one. -@subsection sort_policy_doc_nstut SortPolicy policy class +### `SortPolicy` policy class -The SortPolicy template parameter allows specification of how the NeighborSearch -object will decide which points are to be searched for. The -mlpack::neighbor::NearestNeighborSort class is a well-documented example. A -custom SortPolicy class must implement the same methods which -NearestNeighborSort does: +The `SortPolicy` template parameter allows specification of how the +NeighborSearch object will decide which points are to be searched for. The +`mlpack::neighbor::NearestNeighborSort` class is a well-documented example. A +custom `SortPolicy` class must implement the same methods which +`NearestNeighborSort` does: -@code +```c++ static size_t SortDistance(const arma::vec& list, double newDistance); static bool IsBetter(const double value, const double ref); @@ -360,57 +337,57 @@ static double BestPointToNodeDistance(const arma::vec& queryPoint, static const double WorstDistance(); static const double BestDistance(); -@endcode +``` -The mlpack::neighbor::FurthestNeighborSort class is another implementation, -which is used to create the 'KFN' typedef class, which finds the furthest +The `mlpack::neighbor::FurthestNeighborSort` class is another implementation, +which is used to create the `KFN` typedef class, which finds the furthest neighbors, as opposed to the nearest neighbors. -@subsection metric_type_doc_nstut MetricType policy class +## `MetricType` policy class -The MetricType policy class allows the neighbor search to take place in any -arbitrary metric space. The mlpack::metric::LMetric class is a good example +The `MetricType` policy class allows the neighbor search to take place in any +arbitrary metric space. The `mlpack::metric::LMetric` class is a good example implementation. A MetricType class must provide the following functions: -@code +```c++ // Empty constructor is required. MetricType(); // Compute the distance between two points. template double Evaluate(const VecType& a, const VecType& b); -@endcode +``` -Internally, the NeighborSearch class keeps an instantiated MetricType class +Internally, the `NeighborSearch` class keeps an instantiated `MetricType` class (which can be given in the constructor). This is useful for a metric like the -Mahalanobis distance (mlpack::metric::MahalanobisDistance), which must store +Mahalanobis distance (`mlpack::metric::MahalanobisDistance`), which must store state (the covariance matrix). Therefore, you can write a non-static MetricType -class and use it seamlessly with NeighborSearch. +class and use it seamlessly with `NeighborSearch`. -For more information on the MetricType policy, see the documentation -\ref metrics "here". +For more information on the `MetricType` policy, see the [documentation for +`MetricType`s](../developer/metrics.md). -@subsection mat_type_doc_nstut MatType policy class +### `MatType` policy class -The MatType template parameter specifies the type of data matrix used. This +The `MatType` template parameter specifies the type of data matrix used. This type must implement the same operations as an Armadillo matrix, and so standard -choices are @c arma::mat and @c arma::sp_mat. +choices are `arma::mat` and `arma::sp_mat`. -@subsection tree_type_doc_nstut TreeType policy class +### `TreeType` policy class The NeighborSearch class allows great extensibility in the selection of the type of tree used for search. This type must follow the typical mlpack TreeType -policy, documented \ref trees "here". +policy, documented [here](../developer/trees.md). -Typical choices might include mlpack::tree::KDTree, mlpack::tree::BallTree, -mlpack::tree::StandardCoverTree, mlpack::tree::RTree, or -mlpack::tree::RStarTree. It is easily possible to make your own tree type for -use with NeighborSearch; consult the \ref trees "TreeType documentation" for -more details. +Typical choices might include `mlpack::tree::KDTree`, `mlpack::tree::BallTree`, +`mlpack::tree::StandardCoverTree`, `mlpack::tree::RTree`, or +`mlpack::tree::RStarTree`. It is easily possible to make your own tree type for +use with NeighborSearch; consult the [TreeType +documentation](../developer/trees.md) for more details. -An example of using the NeighborSearch class with a ball tree is given below. +An example of using the `NeighborSearch` class with a ball tree is given below. -@code +```c++ // Construct a NeighborSearch object with ball bounds. NeighborSearch< NearestNeighborSort, @@ -418,43 +395,41 @@ NeighborSearch< arma::mat, tree::BallTree > neighborSearch(dataset); -@endcode +``` -@subsection traverser_type_doc_nstut TraverserType policy class +### `TraverserType` policy class -The last template parameter the NeighborSearch class offers is the TraverserType -class. The TraverserType class holds the strategy used to traverse the trees in -either single-tree or dual-tree search mode. By default, it is set to use the -default traverser of the given @c TreeType (which is the member @c -TreeType::DualTreeTraverser). +The last template parameter the `NeighborSearch` class offers is the +`TraverserType` class. The `TraverserType` class holds the strategy used to +traverse the trees in either single-tree or dual-tree search mode. By default, +it is set to use the default traverser of the given `TreeType` (which is the +member `TreeType::DualTreeTraverser`). This class must implement the following two methods: -@code +```c++ // Instantiate with a given RuleType. TraverserType(RuleType& rule); // Traverse with two trees. void Traverse(TreeType& queryNode, TreeType& referenceNode); -@endcode +``` -The RuleType class provides the following functions for use in the traverser: +The `RuleType` class provides the following functions for use in the traverser: -@code +```c++ // Evaluate the base case between two points. double BaseCase(const size_t queryIndex, const size_t referenceIndex); // Score the two nodes to see if they can be pruned, returning DBL_MAX if they // can be pruned. double Score(TreeType& queryNode, TreeType& referenceNode); -@endcode +``` Note also that any traverser given must satisfy the definition of a pruning dual-tree traversal given in the paper "Tree-independent dual-tree algorithms". -@section further_doc_nstut Further documentation +## Further documentation -For further documentation on the NeighborSearch class, consult the -\ref mlpack::neighbor::NeighborSearch "complete API documentation". - -*/ +For further documentation on the NeighborSearch class, consult the comments in +the source code, found in `mlpack/methods/neighbor_search/`. diff --git a/doc/tutorials/range_search/range_search.txt b/doc/tutorials/range_search.md similarity index 57% rename from doc/tutorials/range_search/range_search.txt rename to doc/tutorials/range_search.md index 645f6233ca..310782b8ea 100644 --- a/doc/tutorials/range_search/range_search.txt +++ b/doc/tutorials/range_search.md @@ -1,17 +1,9 @@ -/*! - -@file range_search.txt -@author Ryan Curtin -@brief Tutorial for how to use the RangeSearch class. - -@page rstutorial RangeSearch tutorial (mlpack_range_search) - -@section intro_rstut Introduction +# RangeSearch tutorial (`mlpack_range_search`) Range search is a simple machine learning task which aims to find all the neighbors of a point that fall into a certain range of distances. In this -setting, we have a \b query and a \b reference dataset. Given a certain range, -for each point in the \b query dataset, we wish to know all points in the \b +setting, we have a *query* and a *reference* dataset. Given a certain range, +for each point in the *query* dataset, we wish to know all points in the \b reference dataset which have distances within that given range to the given query point. @@ -19,60 +11,45 @@ Alternately, if the query and reference datasets are the same, the problem can be stated more simply: for each point in the dataset, we wish to know all points which have distance in the given range to that point. -\b mlpack provides: +mlpack provides: - - a \ref cli_rstut "simple command-line executable" to run range search - - a \ref rs_rstut "simple C++ interface" to perform range search - - a \ref rs_ext_rstut "generic, extensible, and powerful C++ class (RangeSearch)" for complex usage + - a simple command-line executable to run range search + - a simple C++ interface to perform range search + - a generic, extensible, and powerful C++ class (`RangeSearch`) for complex + usage -@section toc_rstut Table of Contents +## The `mlpack_range_search` command-line executable -A list of all the sections this tutorial contains. +mlpack provides a command-line program, `mlpack_range_search`, which can be used +to perform range searches quickly and simply. *(Note that unlike other +bindings, a range search binding is not currently available in other languages +that mlpack provides bindings to.)* - - \ref intro_rstut - - \ref toc_rstut - - \ref cli_rstut - - \ref cli_ex1_rstut - - \ref cli_ex2_rstut - - \ref cli_ex3_rstut - - \ref rs_rstut - - \ref rs_ex1_rstut - - \ref rs_ex2_rstut - - \ref rs_ex3_rstut - - \ref rs_ext_rstut - - \ref metric_type_doc_rstut - - \ref mat_type_doc_rstut - - \ref tree_type_doc_rstut - - \ref further_doc_rstut - -@section cli_rstut The 'mlpack_range_search' command-line executable - -\b mlpack provides an executable, \c mlpack_range_search, which can be used to -perform range searches quickly and simply from the command-line. This program -will perform the range search and place the resulting neighbor index list into -one file and their corresponding distances into another file. These files are -organized such that the first row corresponds to the neighbors (or distances) of -the first query point, and the second row corresponds to the neighbors (or -distances) of the second query point, and so forth. The neighbors of a specific -point are not arranged in any specific order. +The `mlpack_range_search` program will perform the range search and place the +resulting neighbor index list into one file and their corresponding distances +into another file. These files are organized such that the first row +corresponds to the neighbors (or distances) of the first query point, and the +second row corresponds to the neighbors (or distances) of the second query +point, and so forth. The neighbors of a specific point are not arranged in any +specific order. Because a range search may return different numbers of points (including zero), the output file is technically not a valid CSV and may not be loadable by other programs. Therefore, if you need the results in a certain format, it may be -better to use the \ref rs_rstut "C++ interface" to manually export the data in -the preferred format. +better to use the C++ interface to manually export the data in the preferred +format. -Below are several examples of simple usage (and the resultant output). The '-v' +Below are several examples of simple usage (and the resultant output). The `-v` option is used so that output is given. Further documentation on each individual option can be found by typing -@code +```sh $ mlpack_range_search --help -@endcode +``` -@subsection cli_ex1_rstut One dataset, points with distance <= 0.01 +### One dataset, points with distance <= 0.01 -@code +```sh $ mlpack_range_search -r dataset.csv -n neighbors_out.csv -d distances_out.csv \ > -U 0.076 -v [INFO ] Loading 'dataset.csv' as CSV data. Size is 3 x 1000. @@ -108,13 +85,13 @@ search... [INFO ] range_search/computing_neighbors: 0.017110s [INFO ] total_time: 0.033313s [INFO ] tree_building: 0.002500s -@endcode +``` Convenient program timers are given for different parts of the calculation at the bottom of the output, as well as the parameters the simulation was run with. Now, if we look at the output files: -@code +```sh $ head neighbors_out.csv 862 703 @@ -138,15 +115,15 @@ $ head distances_out.csv 0.0700532 0.0529565, 0.0550988 0.0447142, 0.0399286, 0.0734605 -@endcode +``` We can see that only point 862 is within distance 0.076 of point 0. We can -also see that point 2 has no points within a distance of 0.076 -- that line is +also see that point 2 has no points within a distance of 0.076---that line is empty. -@subsection cli_ex2_rstut Query and reference dataset, range [1.0, 1.5] +### Query and reference dataset, range `[1.0, 1.5]` -@code +```sh $ mlpack_range_search -q query_dataset.csv -r reference_dataset.csv -n \ > neighbors_out.csv -d distances_out.csv -L 1.0 -U 1.5 -v [INFO ] Loading 'reference_dataset.csv' as CSV data. Size is 3 x 1000. @@ -185,17 +162,17 @@ $ mlpack_range_search -q query_dataset.csv -r reference_dataset.csv -n \ [INFO ] range_search/computing_neighbors: 0.024427s [INFO ] total_time: 0.045403s [INFO ] tree_building: 0.003979s -@endcode +``` -@subsection cli_ex3_rstut One dataset, range [0.7 0.8], leaf size of 15 points +### One dataset, range `[0.7, 0.8]`, leaf size of 15 points -The \b mlpack implementation of range search is a dual-tree algorithm; when -\f$kd\f$-trees are used, the leaf size of the tree can be changed. Depending on -the characteristics of the dataset, a larger or smaller leaf size can provide -faster computation. The leaf size is modifiable through the command-line -interface, as shown below. +The mlpack implementation of range search is a dual-tree algorithm; when +`kd`-trees are used, the leaf size of the tree can be changed. Depending on the +characteristics of the dataset, a larger or smaller leaf size can provide faster +computation. The leaf size is modifiable through the command-line interface, as +shown below. -@code +```sh $ mlpack_range_search -r dataset.csv -n neighbors_out.csv -d distances_out.csv \ > -L 0.7 -U 0.8 -l 15 -v [INFO ] Loading 'dataset.csv' as CSV data. Size is 3 x 1000. @@ -231,32 +208,33 @@ search... [INFO ] range_search/computing_neighbors: 0.411041s [INFO ] total_time: 0.539931s [INFO ] tree_building: 0.004695s -@endcode +``` -Further documentation on options should be found by using the --help option. +Further documentation on options should be found by using the `--help` option. -@section rs_rstut The 'RangeSearch' class +## The `RangeSearch` class -The 'RangeSearch' class is an extensible template class which allows a high +The `RangeSearch` class is an extensible template class which allows a high level of flexibility. However, all of the template arguments have default -parameters, allowing a user to simply use 'RangeSearch<>' for simple usage +parameters, allowing a user to simply use `RangeSearch<>` for simple usage without worrying about the exact necessary template parameters. -The class bears many similarities to the \ref nstutorial "NeighborSearch" class; -usage generally consists of calling the constructor with one or two datasets, -and then calling the 'Search()' method to perform the actual range search. +The class bears many similarities to the [`NeighborSearch`](neighbor_search.md) +class; usage generally consists of calling the constructor with one or two +datasets, and then calling the `Search()` method to perform the actual range +search. -The 'Search()' method stores the results in two vector-of-vector objects. This +The `Search()` method stores the results in two vector-of-vector objects. This is necessary because each query point may have a different number of neighbors in the specified distance range. The structure of those two objects is very -similar to the output files --neighbors_file and --distances_file for the CLI -interface (see above). A handful of examples of simple usage of the RangeSearch -class are given below. +similar to the output files `--neighbors_file` and `--distances_file` for the +command-line interface (see above). A handful of examples of simple usage of +the `RangeSearch` class are given below. -@subsection rs_ex1_rstut Distance less than 2.0 on a single dataset +### Distance less than `2.0` on a single dataset -@code -#include +```c++ +#include using namespace mlpack::range; @@ -273,14 +251,15 @@ std::vector > resultingDistances; math::Range r(0.0, 2.0); // [0.0, 2.0]. a.Search(r, resultingNeighbors, resultingDistances); -@endcode +``` -The output of the search is stored in resultingNeighbors and resultingDistances. +The output of the search is stored in `resultingNeighbors` and +`resultingDistances`. -@subsection rs_ex2_rstut Range [3.0, 4.0] on a query and reference dataset +### Range `[3.0, 4.0]` on a query and reference dataset -@code -#include +```c++ +#include using namespace mlpack::range; @@ -297,14 +276,14 @@ std::vector > resultingDistances; math::Range r(3.0, 4.0); // [3.0, 4.0]. a.Search(queryData, r, resultingNeighbors, resultingDistances); -@endcode +``` -@subsection rs_ex3_rstut Naive (exhaustive) search for distance greater than 5.0 on one dataset +### Naive (exhaustive) search for distance greater than `5.0` on one dataset -This example uses the O(n^2) naive search (not the tree-based search). +This example uses the `O(n^2)` naive search (not the tree-based search). -@code -#include +```c++ +#include using namespace mlpack::range; @@ -322,81 +301,82 @@ std::vector > resultingDistances; math::Range r(5.0, DBL_MAX); // [5.0, inf). a.Search(r, resultingNeighbors, resultingDistances); -@endcode +``` Needless to say, naive search can be very slow... -@section rs_ext_rstut The extensible 'RangeSearch' class +## The extensible `RangeSearch` class -Similar to the \ref nstutorial "NeighborSearch class", the RangeSearch class is -very extensible, having the following template arguments: +Similar to the [`NeighborSearch` class](neighbor_search.md), the `RangeSearch` +class is very extensible, having the following template arguments: -@code +```c++ template class TreeType = tree::KDTree> class RangeSearch; -@endcode +``` By choosing different components for each of these template classes, a very arbitrary range searching object can be constructed. -@subsection metric_type_doc_rstut MetricType policy class +### `MetricType` policy class -The MetricType policy class allows the range search to take place in any -arbitrary metric space. The mlpack::metric::LMetric class is a good example -implementation. A MetricType class must provide the following functions: +The `MetricType` policy class allows the range search to take place in any +arbitrary metric space. The `mlpack::metric::LMetric` class is a good example +implementation. A `MetricType` class must provide the following functions: -@code +```c++ // Empty constructor is required. MetricType(); // Compute the distance between two points. template double Evaluate(const VecType& a, const VecType& b); -@endcode +``` -Internally, the RangeSearch class keeps an instantiated MetricType class (which -can be given in the constructor). This is useful for a metric like the -Mahalanobis distance (mlpack::metric::MahalanobisDistance), which must store -state (the covariance matrix). Therefore, you can write a non-static MetricType -class and use it seamlessly with RangeSearch. +Internally, the `RangeSearch` class keeps an instantiated `MetricType` class +(which can be given in the constructor). This is useful for a metric like the +Mahalanobis distance (`mlpack::metric::MahalanobisDistance`), which must store +state (the covariance matrix). Therefore, you can write a non-static +`MetricType` class and use it seamlessly with `RangeSearch`. -@subsection mat_type_doc_rstut MatType policy class +See also the [documentation for the `MetricType` +policy](../developer/metrics.md). -The MatType template parameter specifies the type of data matrix used. This +### `MatType` policy class + +The `MatType` template parameter specifies the type of data matrix used. This type must implement the same operations as an Armadillo matrix, and so standard -choices are @c arma::mat and @c arma::sp_mat. +choices are `arma::mat` and `arma::sp_mat`. -@subsection tree_type_doc_rstut TreeType policy class +### `TreeType` policy class -The RangeSearch class also allows a custom tree to be used. The TreeType policy -is also used elsewhere in mlpack and is documented more thoroughly -\ref trees "here". +The `RangeSearch` class also allows a custom tree to be used. The `TreeType` +policy is also used elsewhere in mlpack and is documented more thoroughly +[here](../developer/trees.md). -Typical choices might include mlpack::tree::KDTree (the default), -mlpack::tree::BallTree, mlpack::tree::RTree, mlpack::tree::RStarTree, -or mlpack::tree::StandardCoverTree. Below is an example that uses the -RangeSearch class with an R-tree: +Typical choices might include `mlpack::tree::KDTree` (the default), +`mlpack::tree::BallTree`, `mlpack::tree::RTree`, `mlpack::tree::RStarTree`, or +`mlpack::tree::StandardCoverTree`. Below is an example that uses the +`RangeSearch` class with an R-tree: -@code +```c++ // Construct a RangeSearch object with ball bounds. RangeSearch< metric::EuclideanDistance, arma::mat, tree::RTree > rangeSearch(dataset); -@endcode +``` For further information on trees, including how to write your own tree for use -with RangeSearch and other mlpack methods, see the -\ref trees "TreeType policy documentation". +with `RangeSearch` and other mlpack methods, see the [TreeType policy +documentation](../developer/trees.md). -@section further_doc_rstut Further documentation +## Further documentation -For further documentation on the RangeSearch class, consult the -\ref mlpack::range::RangeSearch "complete API documentation". - -*/ +For further documentation on the `RangeSearch` class, consult the documentation +in the source code, found in `mlpack/methods/range_search/`. diff --git a/doc/tutorials/reinforcement_learning.md b/doc/tutorials/reinforcement_learning.md new file mode 100644 index 0000000000..a0f1e4fb81 --- /dev/null +++ b/doc/tutorials/reinforcement_learning.md @@ -0,0 +1,389 @@ +# Reinforcement Learning Tutorial + +Reinforcement Learning is one of the hottest topics right now, with interest +surging after DeepMind published their article on training deep neural networks +to play Atari games to great success. mlpack implements a complete end-to-end +framework for Reinforcement Learning, featuring multiple environments, policies +and methods. Of course, custom environments and policies can be used and plugged +into the existing framework with no runtime overhead. + +mlpack implements typical benchmark environments (Acrobot, Mountain car etc.), +commonly used policies, replay methods and supports asynchronous learning as +well. In addition, it can [communicate](https://github.com/zoq/gym_tcp_api) with +the OpenAI Gym toolkit for more environments. + +## Reinforcement Learning Environments + +mlpack implements a number of the most popular environments used for testing RL +agents and algorithms. These include the Cart Pole, Acrobot, Mountain Car and +their variations. Of course, as mentioned above, you can communicate with OpenAI +Gym for other environments, like the Atari video games. + +A key component of mlpack is its extensibility. It is a simple process to create +your own custom environments, specific to your needs, and use it with mlpack's +RL framework. All the environments implement a few specific methods and classes +which are used by the agents while learning. + +- `State`: The `State` class is a representation of the environment. For the + `CartPole`, this would involve storing the position, velocity, angle and + angular velocity. + +- `Action`: For discrete environments, `Action` is a class with an enum naming + all the possible actions the agent can take in the environment. Continuing + with the `CartPole` example, the enum would simply contain the two possible + actions, `backward` and `forward`. For continuous environments, the `Action` + class contains an array with its size depending on the action space. + +- `Sample`: This method is perhaps the heart of the environment, providing + rewards to the agent depending on the state and the action taken, and updates + the state based on the action taken as well. + +Of course, your custom environment will most likely make use of a number of +helper methods, depending on your application, such as the `Dsdt` method in the +`Acrobot` environment, used in the `RK4` iterative method (also another helper +method) to estimate the next state. + +## Components of an RL Agent + +A Reinforcement Learning agent, in general, takes actions in an environment in +order to maximize a cumulative reward. To that end, it requires a way to choose +actions (*policy*) and a way to sample previous experiences (*replay*). + +An example of a simple policy would be an epsilon-greedy policy. Using such a +policy, the agent will choose actions greedily with some probability epsilon. +This probability is slowly decreased over time, balancing the line between +exploration and exploitation. + +Similarly, an example of a simple replay would be a random replay. At each time +step, the interactions between the agent and the environment are saved to a +memory buffer and previous experiences are sampled from the buffer to train the +agent. + +Instantiating the components of an agent can be easily done by passing the +Environment as a templated argument and the parameters of the policy/replay to +the constructor. + +To create a Greedy Policy and Prioritized Replay for the `CartPole` environment, +we would do the following: + +```c++ +GreedyPolicy policy(1.0, 1000, 0.1); +PrioritizedReplay replayMethod(10, 10000, 0.6); +``` + +The arguments to `policy` are the initial epsilon values, the interval of +decrease in its value and the value at which epsilon bottoms out and won't be +reduced further. The arguments to `replayMethod` are size of the batch returned, +the number of examples stored in memory, and the degree of prioritization. + +In addition to the above components, an RL agent requires many hyperparameters +to be tuned during it's training period. These parameters include everything +from the discount rate of the future reward to whether Double Q-learning should +be used or not. The `TrainingConfig` class can be instantiated and configured as +follows: + +```c++ +TrainingConfig config; +config.StepSize() = 0.01; +config.Discount() = 0.9; +config.TargetNetworkSyncInterval() = 100; +config.ExplorationSteps() = 100; +config.DoubleQLearning() = false; +config.StepLimit() = 200; +``` + +The object `config` describes an RL agent, using a step size of 0.01 for the +optimization process, a discount factor of 0.9, sync interval of 200 episodes. +This agent only starts learning after storing 100 exploration steps, has a step +limit of 200, and does not utilize double q-learning. + +In this way, we can easily configure an RL agent with the desired +hyperparameters. + +## Q-Learning in mlpack + +Here, we demonstrate Q-Learning in mlpack through the use of a simple example, +the training of a Q-Learning agent on the `CartPole` environment. The code has +been broken into chunks for easy understanding. + +```c++ +#include + +using namespace mlpack; +using namespace mlpack::ann; +using namespace ens; +using namespace mlpack::rl; +``` + +We include all the necessary components of our toy example and declare +namespaces for convenience. + +```c++ +int main() +{ + // Set up the network. + SimpleDQN<> model(4, 64, 32, 2); +``` + +The first step in setting our Q-learning agent is to setup the network for it to +use. `SimpleDQN` class creates a simple feed forward network with 2 hidden +layers. The network constructed here has an input shape of 4 and output shape +of 2. This corresponds to the structure of the `CartPole` environment, where +each state is represented as a column vector with 4 data members (position, +velocity, angle, angular velocity). Similarly, the output shape is represented +by the number of possible actions, which in this case, is only 2 (`foward` and +`backward`). + +We can also use mlpack's ann module to set up a custom `FFN` network. For +example, here we use a single hidden layer. However, the Q-Learning agent +expects the object to have a `ResetNoise` method which `SimpleDQN` has. We +can't pass mlpack's `FFN` network directly. Instead, we have to wrap it into +`SimpleDQN` object. + +```c++ +int main() +{ + // Set up the network. + FFN, GaussianInitialization> network(MeanSquaredError<>(), + GaussianInitialization(0, 0.001)); + network.Add>(4, 128); + network.Add>(); + network.Add>(128, 128); + network.Add>(); + network.Add>(128, 2); + + SimpleDQN<> model(network); + +``` + +The next step would be to setup the other components of the Q-learning agent, +namely its policy, replay method and hyperparameters. + +```c++ + // Set up the policy and replay method. + GreedyPolicy policy(1.0, 1000, 0.1, 0.99); + RandomReplay replayMethod(10, 10000); + + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.9; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 200; +``` + +And now, we get to the heart of the program, declaring a Q-Learning agent. + +```c++ + QLearning + agent(config, model, policy, replayMethod); +``` + +Here, we call the `QLearning` constructor, passing in the type of environment, +network, updater, policy and replay. We use `decltype(var)` as a shorthand for +the variable, saving us the trouble of copying the lengthy templated type. + +We pass references of the objects we created, as parameters to `QLearning` +class. + +Now, we have our Q-Learning agent `agent` ready to be trained on the Cart Pole +environment. + +```c++ + arma::running_stat averageReturn; + size_t episodes = 0; + bool converged = true; + while (true) + { + double episodeReturn = agent.Episode(); + averageReturn(episodeReturn); + episodes += 1; + + if (episodes > 1000) + { + std::cout << "Cart Pole with DQN failed." << std::endl; + converged = false; + break; + } + + /** + * Reaching running average return 35 is enough to show it works. + */ + std::cout << "Average return: " << averageReturn.mean() + << " Episode return: " << episodeReturn << std::endl; + if (averageReturn.mean() > 35) + break; + } + if (converged) + std::cout << "Hooray! Q-Learning agent successfully trained" << std::endl; + + return 0; +} +``` + +We set up a loop to train the agent. The exit condition is determined by the +average reward which can be computed with `arma::running_stat`. It is used for +storing running statistics of scalars, which in this case is the reward signal. +The agent can be said to have converged when the average return reaches a +predetermined value (i.e. > 35). + +Conversely, if the average return does not go beyond that amount even after a +thousand episodes, we can conclude that the agent will not converge and exit the +training loop. + +## Asynchronous Learning + +In 2016, Researchers at Deepmind and University of Montreal published their +paper "Asynchronous Methods for Deep Reinforcement Learning". In it they +described asynchronous variants of four standard reinforcement learning +algorithms: + + - One-Step SARSA + - One-Step Q-Learning + - N-Step Q-Learning + - Advantage Actor-Critic(A3C) + +Online RL algorithms and Deep Neural Networks make an unstable combination +because of the non-stationary and correlated nature of online updates. Although +this is solved by Experience Replay, it has several drawbacks: it uses more +memory and computation per real interaction; and it requires off-policy learning +algorithms. + +Asynchronous methods, instead of experience replay, asynchronously executes +multiple agents in parallel, on multiple instances of the environment, which +solves all the above problems. + +Here, we demonstrate Asynchronous Learning methods in mlpack through the +training of an async agent. Asynchronous learning involves training several +agents simultaneously. Here, each of the agents are referred to as "workers". +Currently mlpack has One-Step Q-Learning worker, N-Step Q-Learning worker and +One-Step SARSA worker. + +Let's examine the sample code in chunks. + +Here we don't use experience replay, and instead of a single policy, we use +three different policies, each corresponding to its worker. Number of workers +created, depends on the number of policies given in the Aggregated Policy. The +column vector contains the probability distribution for each child policy. We +should make sure its size is same as the number of policies and the sum of its +elements is equal to 1. + +``` +AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), + GreedyPolicy(0.7, 5000, 0.01), + GreedyPolicy(0.7, 5000, 0.5)}, + arma::colvec("0.4 0.3 0.3")); +``` + +Now, we will create the `OneStepQLearning` agent. We could have used +`NStepQLearning` or `OneStepSarsa` here according to our requirement. + +```c++ +OneStepQLearning + agent(std::move(config), std::move(model), std::move(policy)); +``` + +Here, unlike the Q-Learning example, instead of the entire while loop, we use +the `Train()` method of the Asynchronous Learning class inside a for loop. 100 +training episodes will take around 50 seconds. + +```c++ +for (int i = 0; i < 100; i++) +{ + agent.Train(measure); +} +``` + +What is "measure" here? It is a lambda function which returns a boolean value +(indicating the end of training) and accepts the episode return (total reward of +a deterministic test episode) as parameter. So, let's create that. + +```c++ +arma::vec returns(20, arma::fill::zeros); +size_t position = 0; +size_t episode = 0; + +auto measure = [&returns, &position, &episode](double episodeReturn) +{ + if(episode > 10000) return true; + + returns[position++] = episodeReturn; + position = position % returns.n_elem; + episode++; + + std::cout << "Episode No.: " << episode + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << std::endl; +}; +``` + +This will train three different agents on three CPU threads asynchronously and +use this data to update the action value estimate. + +Voila, that's all there is to it. + +Here is the full code to try this right away: + +```c++ +#include + +using namespace mlpack; +using namespace mlpack::ann; +using namespace mlpack::rl; + +int main() +{ + // Set up the network. + FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); + model.Add>(4, 128); + model.Add>(); + model.Add>(128, 128); + model.Add>(); + model.Add>(128, 2); + + AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), + GreedyPolicy(0.7, 5000, 0.01), + GreedyPolicy(0.7, 5000, 0.5)}, + arma::colvec("0.4 0.3 0.3")); + + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.9; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 200; + + OneStepQLearning + agent(std::move(config), std::move(model), std::move(policy)); + + arma::vec returns(20, arma::fill::zeros); + size_t position = 0; + size_t episode = 0; + + auto measure = [&returns, &position, &episode](double episodeReturn) + { + if(episode > 10000) return true; + + returns[position++] = episodeReturn; + position = position % returns.n_elem; + episode++; + + std::cout << "Episode No.: " << episode + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << std::endl; + }; + + for (int i = 0; i < 100; i++) + { + agent.Train(measure); + } +} +``` + +## Further Documentation + +For further documentation on the reinforcement learning classes, consult the +documentation in the source code, found in +`mlpack/methods/reinforcement_learning/`. diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt deleted file mode 100644 index e53edd0c74..0000000000 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ /dev/null @@ -1,410 +0,0 @@ -/*! -@file reinforcement_learning.txt -@author Sriram S K -@author Joel Joseph -@brief Tutorial for how to use the Reinforcement Learning module in mlpack. - -@page rltutorial Reinforcement Learning Tutorial - -@section intro_rltut Introduction - -Reinforcement Learning is one of the hottest topics right now, with -interest surging after DeepMind published their article on training -deep neural networks to play Atari games to great success. mlpack -implements a complete end-to-end framework for Reinforcement Learning, -featuring multiple environments, policies and methods. Of course, -custom environments and policies can be used and plugged into the -existing framework with no runtime overhead. - -mlpack implements typical benchmark environments (Acrobot, Mountain car etc.), -commonly used policies, replay methods and supports asynchronous -learning as well. In addition, it can [communicate](https://github.com/zoq/gym_tcp_api) -with the OpenAI Gym toolkit for more environments. - -@section toc_rltut Table of Contents - -This tutorial is split into the following sections: - - - \ref intro_rltut - - \ref toc_rltut - - \ref environment_rltut - - \ref agent_components_rltut - - \ref q_learning_rltut - - \ref async_learning_rltut - - \ref further_rltut - -@section environment_rltut Reinforcement Learning Environments - -mlpack implements a number of the most popular environments used for testing -RL agents and algorithms. These include the Cart Pole, Acrobot, Mountain Car -and their variations. Of course, as mentioned above, you can communicate with -OpenAI Gym for other environments, like the Atari video games. - -A key component of mlpack is its extensibility. It is a simple process to create -your own custom environments, specific to your needs, and use it with mlpack's -RL framework. All the environments implement a few specific methods and classes -which are used by the agents while learning. - -- \c State: The State class is a representation of the environment. For the CartPole, - this would involve storing the position, velocity, angle and angular velocity. - -- \c Action: For discrete environments, Action is a class with an enum naming all the possible - actions the agent can take in the environment. Continuing with the CartPole example, the enum - would simply contain the two possible actions, `backward` and `forward`. For continuous environments, - the Action class contains an array with its size depending on the action space. - -- \c Sample: This method is perhaps the heart of the environment, providing rewards to - the agent depending on the state and the action taken, and updates the state based on - the action taken as well. - -Of course, your custom environment will most likely make use of a number of helper methods, depending -on your application, such as the \c Dsdt method in the \c Acrobot environment, used in the \c RK4 -iterative method (also another helper method) to estimate the next state. - -@section agent_components_rltut Components of an RL Agent - -A Reinforcement Learning agent, in general, takes actions in an environment in order -to maximize a cumulative reward. To that end, it requires a way to choose actions (\b policy) -and a way to sample previous experiences (\b replay). - -An example of a simple policy would be an epsilon-greedy policy. Using such a policy, the agent -will choose actions greedily with some probability epsilon. This probability is slowly decreased -over time, balancing the line between exploration and exploitation. - -Similarly, an example of a simple replay would be a random replay. At each time step, the -interactions between the agent and the environment are saved to a memory buffer and previous -experiences are sampled from the buffer to train the agent. - -Instantiating the components of an agent can be easily done by passing the Environment as -a templated argument and the parameters of the policy/replay to the constructor. - -To create a Greedy Policy and Prioritized Replay for the CartPole environment, we would do the -following: - -@code -GreedyPolicy policy(1.0, 1000, 0.1); -PrioritizedReplay replayMethod(10, 10000, 0.6); -@endcode - -The arguments to `policy` are the initial epsilon values, the interval of decrease in its value -and the value at which epsilon bottoms out and won't be reduced further. The arguments to -`replayMethod` are size of the batch returned, the number of examples stored in memory, and the -degree of prioritization. - -In addition to the above components, an RL agent requires many hyperparameters to be tuned during - it's training period. These parameters include everything from the discount rate of the future -reward to whether Double Q-learning should be used or not. The `TrainingConfig` class can be -instantiated and configured as follows: - -@code - TrainingConfig config; - config.StepSize() = 0.01; - config.Discount() = 0.9; - config.TargetNetworkSyncInterval() = 100; - config.ExplorationSteps() = 100; - config.DoubleQLearning() = false; - config.StepLimit() = 200; -@endcode - -The object `config` describes an RL agent, using a step size of 0.01 for the optimization process, -a discount factor of 0.9, sync interval of 200 episodes. This agent only starts learning after storing -100 exploration steps, has a step limit of 200, and does not utilize double q-learning. - -In this way, we can easily configure an RL agent with the desired hyperparameters. - -@section q_learning_rltut Q-Learning in mlpack - -Here, we demonstrate Q-Learning in mlpack through the use of a simple example, the training of a Q-Learning -agent on the CartPole environment. The code has been broken into chunks for easy understanding. - -@code -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace mlpack; -using namespace mlpack::ann; -using namespace ens; -using namespace mlpack::rl; -@endcode - -We include all the necessary components of our toy example and declare namespaces for convenience. - -@code -int main() -{ - // Set up the network. - SimpleDQN<> model(4, 64, 32, 2); -@endcode - -The first step in setting our Q-learning agent is to setup the network for it to use. SimpleDQN class creates a -simple feed forward network with 2 hidden layers. The network constructed here has an input shape of 4 and -output shape of 2. This corresponds to the structure of the CartPole environment, where each state is -represented as a column vector with 4 data members (position, velocity, angle, angular velocity). Similarly, -the output shape is represented by the number of possible actions, which in this case, is only 2 -(`foward` and `backward`). - -We can also use mlpack's ann module to setup a custom FFN network. For example, here we use a single -hidden layer. However, the Q-Learning agent expects the object to have a `ResetNoise` method which `SimpleDQN` has. -We can't pass mlpack's FFN network directly. Instead, we have to wrap it into `SimpleDQN` object. - -@code -int main() -{ - // Set up the network. - FFN, GaussianInitialization> network(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); - network.Add>(4, 128); - network.Add>(); - network.Add>(128, 128); - network.Add>(); - network.Add>(128, 2); - - SimpleDQN<> model(network); - -@endcode - -The next step would be to setup the other components of the Q-learning agent, namely its policy, replay -method and hyperparameters. - -@code - // Set up the policy and replay method. - GreedyPolicy policy(1.0, 1000, 0.1, 0.99); - RandomReplay replayMethod(10, 10000); - - TrainingConfig config; - config.StepSize() = 0.01; - config.Discount() = 0.9; - config.TargetNetworkSyncInterval() = 100; - config.ExplorationSteps() = 100; - config.DoubleQLearning() = false; - config.StepLimit() = 200; -@endcode - -And now, we get to the heart of the program, declaring a Q-Learning agent. - -@code - QLearning - agent(config, model, policy, replayMethod); -@endcode - -Here, we call the `QLearning` constructor, passing in the type of environment, -network, updater, policy and replay. We use `decltype(var)` as a shorthand for -the variable, saving us the trouble of copying the lengthy templated type. - -We pass references of the objects we created, as parameters to QLearning class. - -Now, we have our Q-Learning agent `agent` ready to be trained on the Cart Pole environment. - -@code - arma::running_stat averageReturn; - size_t episodes = 0; - bool converged = true; - while (true) - { - double episodeReturn = agent.Episode(); - averageReturn(episodeReturn); - episodes += 1; - - if (episodes > 1000) - { - std::cout << "Cart Pole with DQN failed." << std::endl; - converged = false; - break; - } - - /** - * Reaching running average return 35 is enough to show it works. - */ - std::cout << "Average return: " << averageReturn.mean() - << " Episode return: " << episodeReturn << std::endl; - if (averageReturn.mean() > 35) - break; - } - if (converged) - std::cout << "Hooray! Q-Learning agent successfully trained" << std::endl; - - return 0; -} -@endcode - -We set up a loop to train the agent. The exit condition is determined by the average -reward which can be computed with `arma::running_stat`. It is used for storing running -statistics of scalars, which in this case is the reward signal. The agent can be said -to have converged when the average return reaches a predetermined value (i.e. > 35). - -Conversely, if the average return does not go beyond that amount even after a thousand -episodes, we can conclude that the agent will not converge and exit the training loop. - -@section async_learning_rltut - -In 2016, Researchers at Deepmind and University of Montreal published their paper -"Asynchronous Methods for Deep Reinforcement Learning". In it they described asynchronous -variants of four standard reinforcement learning algorithms: - - One-Step SARSA - - One-Step Q-Learning - - N-Step Q-Learning - - Advantage Actor-Critic(A3C) - -Online RL algorithms and Deep Neural Networks make an unstable combination because of the -non-stationary and correlated nature of online updates. Although this is solved by Experience Replay, -it has several drawbacks: it uses more memory and computation per real interaction; and it requires -off-policy learning algorithms. - -Asynchronous methods, instead of experience replay, asynchronously executes multiple agents -in parallel, on multiple instances of the environment, which solves all the above problems. - -Here, we demonstrate Asynchronous Learning methods in mlpack through the training of an async -agent. Asynchronous learning involves training several agents simultaneously. Here, each of the -agents are referred to as "workers". Currently mlpack has One-Step Q-Learning worker, N-Step -Q-Learning worker and One-Step SARSA worker. - -Let's examine the sample code in chunks. - -Apart from the includes used for the q-learning example, two more have to be included: - -@code -#include -#include -@endcode - -Here we don't use experience replay, and instead of a single policy, we use three different -policies, each corresponding to its worker. Number of workers created, depends on the number of -policies given in the Aggregated Policy. The column vector contains the probability distribution -for each child policy. We should make sure its size is same as the number of policies and the sum -of its elements is equal to 1. - -@code -AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), - GreedyPolicy(0.7, 5000, 0.01), - GreedyPolicy(0.7, 5000, 0.5)}, - arma::colvec("0.4 0.3 0.3")); -@endcode - -Now, we will create the "OneStepQLearning" agent. We could have used "NStepQLearning" or "OneStepSarsa" -here according to our requirement. - -@code -OneStepQLearning - agent(std::move(config), std::move(model), std::move(policy)); -@endcode - -Here, unlike the Q-Learning example, instead of the entire while loop, we use the Train method of the Asynchronous -Learning class inside a for loop. 100 training episodes will take around 50 seconds. - -@code -for (int i = 0; i < 100; i++) -{ - agent.Train(measure); -} -@endcode - -What is "measure" here? It is a lambda function which returns a boolean value (indicating the end of training) -and accepts the episode return (total reward of a deterministic test episode) as parameter. -So, let's create that. - -@code -arma::vec returns(20, arma::fill::zeros); -size_t position = 0; -size_t episode = 0; - -auto measure = [&returns, &position, &episode](double episodeReturn) -{ - if(episode > 10000) return true; - - returns[position++] = episodeReturn; - position = position % returns.n_elem; - episode++; - - std::cout << "Episode No.: " << episode - << "; Episode Return: " << episodeReturn - << "; Average Return: " << arma::mean(returns) << std::endl; -}; -@endcode - -This will train three different agents on three CPU threads asynchronously and use this data to update the -action value estimate. -Voila, thats all there is to it. - -Here is the full code to try this right away: - -@code -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace mlpack; -using namespace mlpack::ann; -using namespace mlpack::rl; -int main() -{ - // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); - model.Add>(4, 128); - model.Add>(); - model.Add>(128, 128); - model.Add>(); - model.Add>(128, 2); - - AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), - GreedyPolicy(0.7, 5000, 0.01), - GreedyPolicy(0.7, 5000, 0.5)}, - arma::colvec("0.4 0.3 0.3")); - - TrainingConfig config; - config.StepSize() = 0.01; - config.Discount() = 0.9; - config.TargetNetworkSyncInterval() = 100; - config.ExplorationSteps() = 100; - config.DoubleQLearning() = false; - config.StepLimit() = 200; - - OneStepQLearning - agent(std::move(config), std::move(model), std::move(policy)); - - arma::vec returns(20, arma::fill::zeros); - size_t position = 0; - size_t episode = 0; - - auto measure = [&returns, &position, &episode](double episodeReturn) - { - if(episode > 10000) return true; - - returns[position++] = episodeReturn; - position = position % returns.n_elem; - episode++; - - std::cout << "Episode No.: " << episode - << "; Episode Return: " << episodeReturn - << "; Average Return: " << arma::mean(returns) << std::endl; - }; - - for (int i = 0; i < 100; i++) - { - agent.Train(measure); - } -} -@endcode - -@section further_rltut Further documentation - -For further documentation on the rl classes, consult the \ref mlpack::rl -"complete API documentation". - -*/ diff --git a/doc/tutorials/tutorials.txt b/doc/tutorials/tutorials.txt deleted file mode 100644 index 1fda337155..0000000000 --- a/doc/tutorials/tutorials.txt +++ /dev/null @@ -1,75 +0,0 @@ -/*! - -@file tutorials.txt -@author Ryan Curtin -@brief List of mlpack tutorials. - -@page tutorials Tutorials - -@section quickstart_tut Quickstart Tutorials - -These tutorials give very quick "getting started" examples that you can use to -get started with mlpack in different languages. - - - \ref python_quickstart - - \ref cli_quickstart - - \ref julia_quickstart - - \ref go_quickstart - - \ref r_quickstart - -@section introd_tut Introductory Tutorials - -These tutorials introduce the basic concepts of working with mlpack, aimed at -developers who want to use and contribute to mlpack but are not sure where to -start. - - - \ref build - - \ref build_windows - - \ref formatdoc - - \ref matrices - - \ref iodoc - - \ref timer - - \ref sample - - \ref sample_ml_app - -@section method_tut Method-specific Tutorials - -These tutorials introduce the various methods mlpack offers, aimed at users who -want to get started quickly. These tutorials start with simple examples and -progress to complex, extensible uses. - - - \ref nstutorial - - \ref lrtutorial - - \ref rstutorial - - \ref dettutorial - - \ref kmtutorial - - \ref fmkstutorial - - \ref emst_tutorial - - \ref amftutorial - - \ref cftutorial - - \ref akfntutorial - - \ref anntutorial - - \ref rltutorial - -@section adv_tut Advanced Tutorials - -These tutorials discuss some of the more advanced functionality contained in -mlpack. - - - \ref bindings - - \ref cv - - \ref hpt_guide - - \ref datasetmapper - -@section policy_tut Policy Class Documentation - -mlpack uses templates to achieve its genericity and flexibility. Some of the -template types used by mlpack are common across multiple machine learning -algorithms. The links below provide documentation for some of these common -types. - - - \ref metrics - - \ref kernels - - \ref trees - -*/ diff --git a/doc/user/build_windows.md b/doc/user/build_windows.md new file mode 100644 index 0000000000..f5cc05e79c --- /dev/null +++ b/doc/user/build_windows.md @@ -0,0 +1,262 @@ +# Building mlpack from source on Windows + +*by German Lancioni, Miguel Canteras, Shikhar Jaiswal, Ziyang Jiang* + +This tutorial will show you how to build mlpack for Windows from source, so +you can later create your own C++ applications, using two different ways: + + - Using CMake to generate an intermeditate Visual Studio solution (`.sln`). + - Use Visual Studio's CMake integration to directly build from the + `CMakeLists.txt`. + +Before you try building mlpack, you may +want to install mlpack using `vcpkg` for Windows. If you don't want to install +using `vcpkg`, skip this section and continue with the build tutorial. + +- Install Git (https://git-scm.com/downloads and execute setup) + +- Install CMake (https://cmake.org/ and execute setup) + +- Install vcpkg (https://github.com/Microsoft/vcpkg and execute setup) + +- To install the mlpack library only: + +``` +PS> .\vcpkg install mlpack:x64-windows +``` + +- To install mlpack and its console programs: + +``` +PS> .\vcpkg install mlpack[tools]:x64-windows +``` + +After installing, in Visual Studio, you can create a new project (or open +an existing one). The library is immediately ready to be included +(via preprocessor directives) and used in your project without additional +configuration. + +## Build Environment + +This tutorial has been designed and tested using: + +- Windows 10 +- Visual Studio 2019 (toolset v142) +- mlpack +- OpenBLAS.0.2.14.1 +- armadillo (newest version) +- and x64 configuration + +The directories and paths used in this tutorial are just for reference purposes. + +## Pre-requisites + +- Install CMake for Windows (win64-x64 version from https://cmake.org/download/) + and make sure you can use it from the Command Prompt (may need to add the + `PATH` to system environment variables or manually set the `PATH` before + running CMake) + +- Download the latest mlpack release from the + [mlpack website](https://www.mlpack.org) + +## Windows build instructions + +- Unzip mlpack to `C:\mlpack\mlpack` +- Open Visual Studio and select: File > New > Project from Existing Code + - Type of project: Visual C++ + - Project location: `C:\mlpack\mlpack` + - Project name: mlpack + - Finish +- Make sure the solution configuration is `Debug` and the solution platform is + `x64` for this Visual Studio project +- We will use this Visual Studio project to get the OpenBLAS dependency in the + next section + +## Dependencies + +### OpenBLAS Dependency + +- Open the NuGet packages manager (Tools > NuGet Package Manager > Manage NuGet + Packages for Solution...) +- Click on the "Browse" tab and search for "openblas" +- Click on OpenBlas and check the mlpack project, then click Install +- Once it has finished installing, close Visual Studio + +### Building OpenBLAS from Source + +Unfortunately, the support for building `LAPACK` and `BLAS` on Windows is quite +poor, due to the need for Fortran compiler and libraries. The easiest method to +get the necessary `BLAS/LAPACK` libraries built on Windows is to compile +OpenBLAS with LLVM's `clang-cl` and `flang` to produce the required static +library (`.lib`) files +compatible with the MSVC compiler. A comprehensive guide on the +compilation of OpenBLAS for Windows can be found +[here](https://github.com/xianyi/OpenBLAS/wiki/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio). + +One could always download prebuilt `LAPACK` and `BLAS` libraries for Windows. +However, there are few official sources, and some of those libraries may require +further `dll`s at runtime which may not be available in your system. + +It you choose to build `OpenBLAS` from source, make sure that `LAPACK` functions +are also built. Finally, make sure that the `openblas.lib` library is linked in +your `Armadillo` build (see below), as well as the library path used for the +CMake options `BLAS_LIBRARIES` and `LAPACK_LIBRARIES` in the mlpack CMake +project. + +### Armadillo Dependency + +- Download the newest version of Armadillo from + [Sourceforge](http://arma.sourceforge.net/download.html) +- Unzip to `C:\mlpack\armadillo` +- Create a `build` directory into `C:\mlpack\armadillo\` +- Open the Command Prompt and navigate to `C:\mlpack\armadillo\build` +- Run CMake: + +``` +cmake -G "Visual Studio 16 2019" -A x64 -DBLAS_LIBRARY:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARY:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" .. +``` + +*Note*: If you are using different directory paths, a different configuration +(e.g. Release) or a different VS version, update the cmake command accordingly. +If CMake cannot identify the compiler version, check if the Visual Studio +compiler and Windows SDK are installed correctly. + +- Once it has successfully finished, open + `C:\mlpack\armadillo\build\armadillo.sln` +- Build > Build Solution +- Once it has successfully finished, close Visual Studio + +## Building mlpack with CMake-generated Solution + +- Create a `build` directory into `C:\mlpack\mlpack\` +- You can generate the project using either cmake via command line or GUI. If + you prefer to use GUI, refer to the appendix +- To use the CMake command line prompt, open the Command Prompt and navigate to + `C:\mlpack\mlpack\build` +- Run cmake: + +``` +cmake -G "Visual Studio 16 2019" -A x64 -DBLAS_LIBRARIES:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARIES:FILEPATH="C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DARMADILLO_INCLUDE_DIR="C:/mlpack/armadillo/include" -DARMADILLO_LIBRARY:FILEPATH="C:/mlpack/armadillo/build/Debug/armadillo.lib" -DDEBUG=OFF -DPROFILE=OFF .. +``` + +*Note*: CMake will attempt to automatically download the ensmallen dependency. +If for some reason cmake can't download the dependency, you will need to +manually download ensmallen from http://ensmallen.org/ and extract it to +`C:\mlpack\mlpack\deps\`. Then, specify the path to ensmallen using the flag: +`-DENSMALLEN_INCLUDE_DIR=C:/mlpack/mlpack/deps/ensmallen/include`. + +- Once CMake configuration has successfully finished, open + `C:\mlpack\mlpack\build\mlpack.sln` +- Build > Build Solution (this may be by default in Debug mode) +- Once it has sucessfully finished, you will find the library files you need in: + `C:\mlpack\mlpack\build\Debug` (or `C:\mlpack\mlpack\build\Release` if you + changed to Release mode) + +You are ready to create your first application; take a look at the +[Sample C++ ML App](sample_ml_app.md). + +## Building mlpack with Visual Studio's CMake integration + +This project can be directly built from the `CMakeLists.txt` with the latest +version of MS Visual Studio, given you have CMake integration via the +[C++ CMake tools for Windows](https://docs.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio?view=msvc-160). +To open the CMake project with Visual Studio, select File -> Open -> CMake in +the top menu, followed by selecting the root `CMakeLists.txt` located in +mlpack's root directory. + +In order to allow Visual Studio to configure the CMake project, the CMake +configuration json will have to be edited to provide the [relevant options +shown in the `README`](../../README.md#2-dependencies) needed to find all the +dependencies. The options that you must provide to Visual Studio's CMake are: + + - `ARMADILLO_INCLUDE_DIR` + - `ARMADILLO_LIBRARY` + - `CEREAL_INCLUDE_DIR` + - `BLAS_LIBRARIES` + - `LAPACK_LIBRARIES` + +The CMake configuration json can be edited in Visual Studio by right clicking +the root `CMakeLists.txt` in the project view, selecting *CMake settings for +mlpack* and finally clicking on *edit JSON*. Adding a new CMake option can be +done by adding object fields with the following format to the variables array in +the `CMakeSettings.json`: + +``` +{ + "name": "options_name_string", + "value": "options_value_string", + "type" : "{BOOL|FILEPATH|PATH|STRING}" +} +``` + +Here is a full example of the `CMakeSettings.json`file: + +``` +{ + "configurations": [ + { + "name": "x64-Debug (default)", + "generator": "Ninja", + "configurationType": "Debug", + "inheritEnvironments": [ "msvc_x64_x64" ], + "buildRoot": "${projectDir}\\out\\build\\${name}", + "installRoot": "${projectDir}\\out\\install\\${name}", + "cmakeCommandArgs": "", + "buildCommandArgs": "", + "ctestCommandArgs": "", + "variables": [ + { + "name": "ARMADILLO_INCLUDE_DIR", + "value": "PATH/TO/CPP/DEPENDENCY/armadillo-10.1.2/include", + "type": "PATH" + }, + { + "name": "ARMADILLO_LIBBRARY", + "value": "PATH/TO/CPP/DEPENDENCY/armadillo-10.1.2/lib/armadillo.lib", + "type": "PATH" + }, + { + "name": "CEREAL_INCLUDE_DIR", + "value": "PATH/TO/CPP/DEPENDENCY/cereal-1.3.0/include", + "type": "PATH" + }, + { + "name": "BLAS_LIBRARIES", + "value": "PATH/TO/CPP/DEPENDENCY/OpenBLAS/lib/openblas.lib", + "type": "PATH" + }, + { + "name": "LAPACK_LIBRARIES", + "value": "PATH/TO/CPP/DEPENDENCY/OpenBLAS/lib/openblas.lib", + "type": "PATH" + } + ] + } + ] +} +``` + +## Appendix + +If you prefer to use the CMake GUI, follow these instructions: + + - To use the CMake GUI, open "CMake". + - For "Where is the source code:" set `C:\mlpack\mlpack\` + - For "Where to build the binaries:" set `C:\mlpack\mlpack\build` + - Click `Configure` + - If there is an error and Armadillo is not found, try "Add Entry" with the + following variables and reconfigure: + - Name: `ARMADILLO_INCLUDE_DIR`; type `PATH`; value `C:/mlpack/armadillo/include/` + - Name: `ARMADILLO_LIBRARY`; type `FILEPATH`; value `C:/mlpack/armadillo/build/Debug/armadillo.lib` + - Name: `BLAS_LIBRARY`; type `FILEPATH`; value `C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a` + - Name: `LAPACK_LIBRARY`; type `FILEPATH`; value `C:/mlpack/mlpack/packages/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a` + - Once CMake has configured successfully, hit "Generate" to create the `.sln` file. + +## Additional information + +If you are facing issues during the build process of mlpack, you may take a look +at other third-party tutorials for Windows, but they may be out of date: + + * [Github wiki Windows Build page](https://github.com/mlpack/mlpack/wiki/WindowsBuild) + * [Keon's tutorial for mlpack 2.0.3](http://keon.io/mlpack-on-windows) + * [Kirizaki's tutorial for mlpack 2](https://overdosedblog.wordpress.com/2016/08/15/once_again/) diff --git a/doc/user/cv.md b/doc/user/cv.md new file mode 100644 index 0000000000..00b95f95cf --- /dev/null +++ b/doc/user/cv.md @@ -0,0 +1,350 @@ +# Cross-Validation + +mlpack implements cross-validation support for its learning algorithms, for a +variety of performance measures. Cross-validation is useful for determining an +estimate of how well the learner will generalize to un-seen test data. It is a +commonly used part of the data science pipeline. + +In short, given some learner and some performance measure, we wish to get an +average of the performance measure given different splits of the dataset into +training data and validation data. The learner is trained on the training data, +and the performance measure is evaluated on the validation data. + +mlpack currently implements two easy-to-use forms of cross-validation: + + - *simple* cross-validation, where we simply desire the performance measure + on a single split of the data into a training set and validation set + + - *k-fold* cross-validation, where we split the data `k` ways and desire the + average performance measure on each of the `k` splits of the data + +In this tutorial we will see the usage examples and details of the +cross-validation module. Because the cross-validation code is generic and can +be used with any learner and performance measure, any use of the +cross-validation code in mlpack has to be in C++. + +## Simple cross-validation examples + +This section contains examples, in C++, showing the usage of mlpack's simple +cross-validation functionality. + +### 10-fold cross-validation on softmax regression + +Suppose we have some data to train and validate on, as defined below: + +```c++ +// 100-point 6-dimensional random dataset. +arma::mat data = arma::randu(6, 100); +// Random labels in the [0, 4] interval. +arma::Row labels = + arma::randi>(100, arma::distr_param(0, 4)); +size_t numClasses = 5; +``` + +The code above generates an 100-point random 6-dimensional dataset with 5 +classes. + +To run 10-fold cross-validation for softmax regression with accuracy as a +performance measure, we can write the following piece of code. + +```c++ +KFoldCV cv(10, data, labels, numClasses); +double lambda = 0.1; +double softmaxAccuracy = cv.Evaluate(lambda); +``` + +Note that the `Evaluate()` method of `KFoldCV` takes any hyperparameters of an +algorithm---that is, anything that is not `data`, `labels`, `numClasses`, +`datasetInfo`, or `weights` (those last three may not be present for every +algorithm type). To be more specific, in this example the `Evaluate()` method +relies on the following `SoftmaxRegression` constructor: + +```c++ +template +SoftmaxRegression(const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda = 0.0001, + const bool fitIntercept = false, + OptimizerType optimizer = OptimizerType()); +``` + +which has the parameter `lambda` after three conventional arguments (`data`, +\c labels and \c numClasses). We can skip passing `fitIntercept` and +`optimizer` since there are the default values. (Technically, we don't even +need to pass `lambda` since there is a default value.) + +In general to cross-validate you need to specify what machine learning algorithm +and metric you are going to use, and then to pass some conventional data-related +parameters into one of the cross-validation constructors and all other +parameters (which are generally hyperparameters) into the `Evaluate()` method. + +### 10-fold cross-validation on weighted decision trees + +In the following example we will cross-validate `DecisionTree` with weights. +This is very similar to the previous example, except that we also have instance +weights for each point in the dataset. We can generate weights for the dataset +from the previous example with the code below: + +```c++ +// Random weights for every point from the code snippet above. +arma::rowvec weights = arma::randu(1, 100); +``` + +Given those weights for each point, we can now perform cross-validation by also +passing the weights to the constructor of `KFoldCV`: + +```c++ +KFoldCV, Accuracy> cv2(10, data, labels, numClasses, weights); +size_t minimumLeafSize = 8; +double weightedDecisionTreeAccuracy = cv2.Evaluate(minimumLeafSize); +``` + +As with the previous example, internally this call to `cv2.Evaluate()` relies +on the following `DecisionTree` constructor: + +```c++ +template +DecisionTree(MatType&& data, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* + = 0); +``` + +### 10-fold cross-validation with categorical decision trees + +`DecisionTree` models can be constructed in multiple other ways. For example, if +we have a dataset with both categorical and numerical features, we can also +perform cross-validation by using the associated `data::DatasetInfo` object. +Thus, given some `data::DatasetInfo` object called `datasetInfo` (that perhaps +was produced by a call to `data::Load()`), we can perform k-fold +cross-validation in a similar manner to the other examples: + +```c++ +KFoldCV, Accuracy> cv3(10, data, datasetInfo, labels, + numClasses); +double decisionTreeWithDIAccuracy = cv3.Evaluate(minimumLeafSize); +``` + +This particular call to `cv3.Evaluate()` relies on the following `DecisionTree` +constructor: + +```c++ +template +DecisionTree(MatType&& data, + const data::DatasetInfo& datasetInfo, + LabelsType&& labels, + const size_t numClasses, + const size_t minimumLeafSize = 10); +``` + +### Simple cross-validation for linear regression + +`SimpleCV` has the same interface as `KFoldCV`, except it takes as one of its +arguments a proportion (from 0 to 1) of data used as a validation set. For +example, to validate `LinearRegression` with 20% of the data used in the +validation set we can write the following code. + +```c++ +// Random responses for every point from the code snippet in the beginning of +// the tutorial. +arma::rowvec responses = arma::randu(100); + +SimpleCV cv4(0.2, data, responses); +double lrLambda = 0.05; +double lrMSE = cv4.Evaluate(lrLambda); +``` + +## Performance measures + +The cross-validation classes require a performance measure to be specified. +\b mlpack has a number of performance measures implemented; below is a list: + + - `mlpack::cv::Accuracy`: a simple measure of accuracy + - `mlpack::cv::F1`: the F1 score; depends on an averaging strategy + - `mlpack::cv::MSE`: minimum squared error (for regression problems) + - `mlpack::cv::Precision`: the precision, for classification problems + - `mlpack::cv::Recall`: the recall, for classification problems + +In addition, it is not difficult to implement a custom performance measure. A +class following the structure below can be used: + +```c++ +class CustomMeasure +{ + // + // This evaluates the metric given a trained model and a set of data (with + // labels or responses) to evaluate on. The data parameter will be a type of + // Armadillo matrix, and the labels will be the labels that go with the model. + // + // If you know that your model is a classification model (and thus that + // ResponsesType will be arma::Row), it is ok to replace the + // ResponsesType template parameter with arma::Row. + // + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& labels) + { + // Inside the method you should call model.Predict() and compare the + // values with the labels, in order to get the desired performance measure + // and return it. + } +}; +``` + +Once this is implemented, then `CustomMeasure` (or whatever the class is +called) is easy to use as a custom performance measure with `KFoldCV` or +`SimpleCV`. + +## The KFoldCV and SimpleCV classes + +This section provides details about the `KFoldCV` and `SimpleCV` classes. The +cross-validation infrastructure is based on heavy amounts of template +metaprogramming, so that any mlpack learner and any performance measure can be +used. Both classes have two required template parameters and one optional +parameter: + + - `MLAlgorithm`: the type of learner to be used + - `Metric`: the performance measure to be evaluated + - `MatType`: the type of matrix used to store the data + +In addition, there are two more template parameters, but these are automatically +extracted from the given `MLAlgorithm` class, and users should not need to +specify these parameters except when using an unconventional type like +`arma::fmat` for data points. + +The general structure of the `KFoldCV` and `SimpleCV` classes is split into two +parts: + + - The constructor: create the object, and store the data for the `MLAlgorithm` + training. + - The `Evaluate()` method: take any non-data parameters for the `MLAlgorithm` + and calculate the desired performance measure. + +This split is important because it defines the API: all data-related parameters +are passed to the constructor, whereas algorithm hyperparameters are passed to +the `Evaluate()` method. + +### The KFoldCV and SimpleCV constructors + +There are six constructors available for `KFoldCV` and `SimpleCV`, each tailored +for a different learning situation. Each is given below for the `KFoldCV` +class, but the same constructors are also available for the `SimpleCV` class, +with the exception that instead of specifying `k`, the number of folds, the +`SimpleCV` class takes a parameter between 0 and 1 specifying the percentage of +the dataset to use as a validation set. + + - `KFoldCV(k, xs, ys)`: this is for unweighted regression applications and + two-class classification applications; `xs` is the dataset and `ys` + are the responses or labels for each point in the dataset. + + - `KFoldCV(k, xs, ys, numClasses)`: this is for unweighted classification + applications; `xs` is the dataset, `ys` are the class labels for each + data point, and `numClasses` is the number of classes in the dataset. + + - `KFoldCV(k, xs, datasetInfo, ys, numClasses)`: this is for unweighted + categorical/numeric classification applications; `xs` is the dataset, + `datasetInfo` is a `data::DatasetInfo` object that holds the types of + each dimension in the dataset, `ys` are the class labels for each data + point, and `numClasses` is the number of classes in the dataset. + + - `KFoldCV(k, xs, ys, weights)`: this is for weighted regression or + two-class classification applications; `xs` is the dataset, `ys` are + the responses or labels for each point in the dataset, and `weights` + are the weights for each point in the dataset. + + - `KFoldCV(k, xs, ys, numClasses, weights)`: this is for weighted + classification applications; `xs` is the dataset, `ys` are the class + labels for each point in the dataset; `numClasses` is the number of + classes in the dataset, and `weights` holds the weights for each point + in the dataset. + + - `KFoldCV(k, xs, datasetInfo, ys, numClasses, weights)`: this is for + weighted cateogrical/numeric classification applications; `xs` is the + dataset, `datasetInfo` is a `data::DatasetInfo` object that holds the + types of each dimension in the dataset, `ys` are the class labels for + each data point, `numClasses` is the number of classes in each dataset, + and `weights` holds the weights for each point in the dataset. + +Note that the constructor you should use is the constructor that most closely +matches the constructor of the machine learning algorithm you would like +performance measures of. So, for instance, if you are doing multi-class softmax +regression, you could call the constructor `SoftmaxRegression(xs, ys, +numClasses)`. Therefore, for `KFoldCV` you would call the constructor +`KFoldCV(k, xs, ys, numClasses)` and for `SimpleCV` you would call the +constructor `SimpleCV(pct, xs, ys, numClasses)`. + +### The `Evaluate()` method + +The other method that `KFoldCV` and `SimpleCV` have is the method to actually +calculate the performance measure: `Evaluate()`. The `Evaluate()` method takes +any hyperparameters that would follow the data arguments to the constructor or +`Train()` method of the given `MLAlgorithm`. The `Evaluate()` method takes no +more arguments than that, and returns the desired performance measure on the +dataset. + +Therefore, let us suppose that we are interested in cross-validating the +performance of a softmax regression model, and that we have constructed the +appropriate `KFoldCV` object using the code below: + +```c++ +KFoldCV cv(k, data, labels, numClasses); +``` + +The `SoftmaxRegression` class has the constructor + +```c++ +template +SoftmaxRegression(const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda = 0.0001, + const bool fitIntercept = false, + OptimizerType optimizer = OptimizerType()); +``` + +Note that all parameters after are `numClasses` are optional. This means that +we can specify none or any of them in our call to `Evaluate()`. Below is some +example code showing three different ways we can call `Evaluate()` with the `cv` +object from the code snippet above. + +```c++ +// First, call with all defaults. +double result1 = cv.Evaluate(); + +// Next, call with lambda set to 0.1 and fitIntercept set to true. +double result2 = cv.Evaluate(0.1, true); + +// Lastly, create a custom optimizer to use for optimization, and use a lambda +// value of 0.5 and fit no intercept. +optimization::SGD<> sgd(0.05, 50000); // Step size of 0.05, 50k max iterations. +double result3 = cv.Evaluate(0.5, false, sgd); +``` + +The same general idea applies to any `MLAlgorithm`: all hyperparameters must be +passed to the `Evaluate()` method of `KFoldCV` or `SimpleCV`. + +@section cvbasic_further Further references + +For further documentation, please see the source code for each of the relevant +classes: + + - `mlpack::cv::SimpleCV` + - `mlpack::cv::KFoldCV` + - `mlpack::cv::Accuracy` + - `mlpack::cv::F1` + - `mlpack::cv::MSE` + - `mlpack::cv::Precision` + - `mlpack::cv::Recall` + +If you are interested in implementing a different cross-validation strategy than +k-fold cross-validation or simple cross-validation, take a look at the +implementations of each of those classes to guide your implementation. + +In addition, the [hyperparameter tuner](hpt.md) documentation may also be +relevant. diff --git a/doc/guide/formats.hpp b/doc/user/formats.md similarity index 51% rename from doc/guide/formats.hpp rename to doc/user/formats.md index 6a9577449e..bc1ef01646 100644 --- a/doc/guide/formats.hpp +++ b/doc/user/formats.md @@ -1,64 +1,37 @@ -/*! @page formatdoc File formats and loading data in mlpack +# File formats and loading data in mlpack -@section formatintro Introduction - -mlpack supports a wide variety of data (including images) and model formats for use in both its -command-line programs and in C++ programs using mlpack via the -mlpack::data::Load() function. This tutorial discusses the formats that are +mlpack supports a wide variety of data (including images) and model formats for +use in both its command-line programs, and in C++ programs via the +`mlpack::data::Load()` function. This tutorial discusses the formats that are supported and how to use them. -@section toc_tut Table of Contents - -This tutorial is split into the following sections: - - - \ref formatintro - - \ref toc_tut - - Data - - Data Formats - - \ref formatsimple - - \ref formattypes - - \ref formatcpp - - \ref sparseload - - \ref formatcat - - \ref formatcatcpp - - Image Support - - \ref intro_imagetut - - \ref model_api_imagetut - - \ref imageinfo_api_imagetut - - \ref load_api_imagetut - - \ref save_api_imagetut - - Models - - \ref formatmodels - - \ref formatmodelscpp - - \ref formatfinal - -@section formatsimple Simple examples to load data in C++ +## Simple examples to load data in C++ The example code snippets below load data from different formats into an -Armadillo matrix object (\c arma::mat) or model when using C++. +Armadillo matrix object (`arma::mat`) or model when using C++. -@code +```c++ using namespace mlpack; arma::mat matrix1; data::Load("dataset.csv", matrix1); -@endcode +``` -@code +```c++ using namespace mlpack; arma::mat matrix2; data::Load("dataset.bin", matrix2); -@endcode +``` -@code +```c++ using namespace mlpack; arma::mat matrix3; data::Load("dataset.h5", matrix3); -@endcode +``` -@code +```c++ using namespace mlpack; // ARFF loading is a little different, since sometimes mapping has to be done @@ -68,58 +41,58 @@ data::DatasetInfo datasetInfo; data::Load("dataset.arff", matrix4, datasetInfo); // The datasetInfo object now holds information about each dimension. -@endcode +``` -@code +```c++ using namespace mlpack; regression::LogisticRegression lr; data::Load("model.bin", "logistic_regression_model", lr); -@endcode +``` -@section formattypes Supported dataset types +## Supported dataset types Datasets in mlpack are represented internally as sparse or dense numeric -matrices (specifically, as \c arma::mat or \c arma::sp_mat or similar). This +matrices (specifically, as `arma::mat` or `arma::sp_mat` or similar). This means that when datasets are loaded from file, they must be converted to a suitable numeric representation. Therefore, in general, datasets on disk should contain only numeric features in order to be loaded successfully by mlpack. The types of datasets that mlpack can load are roughly the same as the types of matrices that Armadillo can load. However, the load functionality that mlpack -provides only supports loading dense datasets. When datasets are loaded -by mlpack, the file's type is detected using the file's extension. +provides ***only supports loading dense datasets***. When datasets are loaded +by mlpack, ***the file's type is detected using the file's extension***. mlpack supports the following file types: - - csv (comma-separated values), denoted by .csv or .txt - - tsv (tab-separated values), denoted by .tsv, .csv, or .txt - - ASCII (raw ASCII, with space-separated values), denoted by .txt - - Armadillo ASCII (Armadillo's text format with a header), denoted by .txt - - PGM, denoted by .pgm - - PPM, denoted by .ppm - - Armadillo binary, denoted by .bin - - Raw binary, denoted by .bin (note: this will be loaded as - one-dimensional data, which is likely not what is desired.) - - HDF5, denoted by .hdf, .hdf5, .h5, or .he5 (note: HDF5 must be enabled - in the Armadillo configuration) - - ARFF, denoted by .arff (note: this is not supported by all mlpack - command-line programs ; see \ref formatcat) + - csv (comma-separated values), denoted by `.csv` or `.txt` + - tsv (tab-separated values), denoted by `.tsv`, `.csv`, or `.txt` + - ASCII (raw ASCII, with space-separated values), denoted by `.txt` + - Armadillo ASCII (Armadillo's text format with a header), denoted by `.txt` + - PGM, denoted by `.pgm` + - PPM, denoted by `.ppm` + - Armadillo binary, denoted by `.bin` + - Raw binary, denoted by `.bin` ***(note: this will be loaded as + one-dimensional data, which is likely not what is desired.)*** + - HDF5, denoted by `.hdf`, `.hdf5`, `.h5`, or `.he5` ***(note: HDF5 must be + enabled in the Armadillo configuration)*** + - ARFF, denoted by .arff ***(note: this is not supported by all mlpack + command-line programs***; see below) -Datasets that are loaded by mlpack should be stored with one row for -one point and one column for one dimension. Therefore, a dataset -with three two-dimensional points \f$(0, 1)\f$, \f$(3, 1)\f$, and \f$(5, -5)\f$ -would be stored in a csv file as: +Datasets that are loaded by mlpack should be stored with ***one row for one +point*** and ***one column for one dimension***. Therefore, a dataset with +three two-dimensional points `(0, 1)`, `(3, 1)`, and `(5, -5)` would be stored +in a csv file as: -\code +``` 0, 1 3, 1 5, -5 -\endcode +``` As noted earlier, for command-line programs, the format is automatically detected at load time. Therefore, a dataset can be loaded in many ways: -\code +``` $ mlpack_logistic_regression -t dataset.csv -v [INFO ] Loading 'dataset.csv' as CSV data. Size is 32 x 37749. ... @@ -131,76 +104,75 @@ $ mlpack_logistic_regression -t dataset.txt -v $ mlpack_logistic_regression -t dataset.h5 -v [INFO ] Loading 'dataset.h5' as HDF5 data. Size is 32 x 37749. ... -\endcode +``` Similarly, the format to save to is detected by the extension of the given filename. -@section formatcpp Loading simple matrices in C++ +## Loading simple matrices in C++ -When C++ is being written, the mlpack::data::Load() and mlpack::data::Save() +When C++ is being written, the `mlpack::data::Load()` and `mlpack::data::Save()` functions are used to load and save datasets, respectively. These functions -should be preferred over the built-in Armadillo \c .load() and \c .save() +should be preferred over the built-in Armadillo `.load()` and `.save()` functions. Matrices in mlpack are column-major, meaning that each column should correspond to a point in the dataset and each row should correspond to a dimension; for -more information, see \ref matrices. This is at odds with how the data is -stored in files; therefore, a transposition is required during load and save. -The mlpack::data::Load() and mlpack::data::Save() functions do this -automatically (unless otherwise specified), which is why they are preferred over -the Armadillo functions. +more information, see [matrices in mlpack](matrices.md). This is at odds with +how the data is stored in files; therefore, a transposition is required during +load and save. The `mlpack::data::Load()` and `mlpack::data::Save()` functions +do this automatically (unless otherwise specified), which is why they are +preferred over the Armadillo functions. To load a matrix from file, the call is straightforward. After creating a matrix object, the data can be loaded: -\code +```c++ arma::mat dataset; // The data will be loaded into this matrix. mlpack::data::Load("dataset.csv", dataset); -\endcode +``` Saving matrices is equally straightforward. The code below generates a random matrix with 10 points in 3 dimensions and saves it to a file as HDF5. -\code +```c++ // 3 dimensions (rows), with 10 points (columns). arma::mat dataset = arma::randu(3, 10); mlpack::data::Save("dataset.h5", dataset); -\endcode +``` As with the command-line programs, the type of data to be loaded is automatically detected from the filename extension. For more details, see the -mlpack::data::Load() and mlpack::data::Save() documentation. +`mlpack::data::Load()` and `mlpack::data::Save()` documentation. -@section sparseload Dealing with sparse matrices +## Dealing with sparse matrices As mentioned earlier, support for loading sparse matrices in mlpack is not available at this time. To use a sparse matrix with mlpack code, you will have to write a C++ program instead of using any of the command-line tools, because the command-line tools all use dense datasets internally. (There is one -exception: the \c mlpack_cf program, for collaborative filtering, loads sparse -coordinate lists.) +exception: the `mlpack_cf` command-line program, for collaborative filtering, +loads sparse coordinate lists.) -In addition, the \c mlpack::data::Load() function does not support loading any -sparse format; so the best idea is to use undocumented Armadillo functionality -to load coordinate lists. Suppose you have a coordinate list file like the one -below: +In addition, the `mlpack::data::Load()` function does not support loading any +sparse format; so the best idea is to use Armadillo functionality to load +coordinate lists. Suppose you have a coordinate list file like the one below: -\code +```sh $ cat cl.csv 0 0 0.332 1 3 3.126 4 4 1.333 -\endcode +``` This represents a 5x5 matrix with three nonzero elements. We can load this using Armadillo: -\code +```c++ arma::sp_mat matrix; matrix.load("cl.csv", arma::coord_ascii); matrix = matrix.t(); // We must transpose after load! -\endcode +``` The transposition after loading is necessary if the coordinate list is in row-major format (that is, if each row in the matrix represents a point and each @@ -208,7 +180,7 @@ column represents a feature). Be sure that the matrix you use with mlpack methods has points as columns and features as rows! See \ref matrices for more information. -@section formatcat Categorical features and command line programs +## Categorical features and command line programs In some situations it is useful to represent data not just as a numeric matrix but also as categorical data (i.e. with numeric but unordered categories). This @@ -218,28 +190,28 @@ categorical features. In some machine learning situations, such as, e.g., decision trees, categorical data can be used. Categorical data might look like this (in CSV format): -\code +``` 0, 1, "true", 3 5, -2, "false", 5 2, 2, "true", 4 3, -1, "true", 3 4, 4, "not sure", 0 0, 7, "false", 6 -\endcode +``` -In the example above, the third dimension (which takes values "true", "false", -and "not sure") is categorical. mlpack can load and work with this data, but -the strings must be mapped to numbers, because all dataset in mlpack are -represented by Armadillo matrix objects. +In the example above, the third dimension (which takes values `"true"`, +`"false"`, and `"not sure"`) is categorical. mlpack can load and work with this +data, but the strings must be mapped to numbers, because all dataset in mlpack +are represented by Armadillo matrix objects. From the perspective of an mlpack command-line program, this support is transparent; mlpack will attempt to load the data file, and if it detects entries in the file that are not numeric, it will map them to numbers and then print, for each dimension, the number of mappings. For instance, if we run the -\c mlpack_hoeffding_tree program (which supports categorical data) on the -dataset above (stored as dataset.csv), we receive this output during loading: +`mlpack_hoeffding_tree` program (which supports categorical data) on the dataset +above (stored as `dataset.csv`), we receive this output during loading: -\code +```sh $ mlpack_hoeffding_tree -t dataset.csv -l dataset.labels.csv -v [INFO ] Loading 'dataset.csv' as CSV data. Size is 6 x 4. [INFO ] 0 mappings in dimension 0. @@ -247,39 +219,39 @@ $ mlpack_hoeffding_tree -t dataset.csv -l dataset.labels.csv -v [INFO ] 3 mappings in dimension 2. [INFO ] 0 mappings in dimension 3. ... -\endcode +``` -Currently, only the \c mlpack_hoeffding_tree program supports loading -categorical data, and this is also the only program that supports loading an -ARFF dataset. +Currently, only the `mlpack_hoeffding_tree` and `mlpack_decision_tree` programs +supports loading categorical data, and this is also the only program that +supports loading an ARFF dataset. -@section formatcatcpp Categorical features and C++ +## Categorical features and C++ When writing C++, loading categorical data is slightly more tricky: the mappings from strings to integers must be preserved. This is the purpose of the -mlpack::data::DatasetInfo class, which stores these mappings and can be used and -load and save time to apply and de-apply the mappings. +`mlpack::data::DatasetInfo` class, which stores these mappings and can be used +and load and save time to apply and de-apply the mappings. When loading a dataset with categorical data, the overload of -mlpack::data::Load() that takes an mlpack::data::DatasetInfo object should be -used. An example is below: +`mlpack::data::Load()` that takes an `mlpack::data::DatasetInfo` object should +be used. An example is below: -\code +```c++ arma::mat dataset; // Load into this matrix. mlpack::data::DatasetInfo info; // Store information about dataset in this. // Load the ARFF dataset. mlpack::data::Load("dataset.arff", dataset, info); -\endcode +``` -After this load completes, the \c info object will hold the information about -the mappings necessary to load the dataset. It is possible to re-use the -\c DatasetInfo object to load another dataset with the same mappings. This is +After this load completes, the `info` object will hold the information about the +mappings necessary to load the dataset. It is possible to re-use the +`DatasetInfo` object to load another dataset with the same mappings. This is useful when, for instance, both a training and test set are being loaded, and it is necessary that the mappings from strings to integers for categorical features are identical. An example is given below. -\code +```c++ arma::mat trainingData; // Load training data into this matrix. mlpack::data::DatasetInfo info; // This will store the mappings. @@ -289,15 +261,15 @@ mlpack::data::Load("training_data.arff", trainingData, info); // Load the test data, but re-use the 'info' object with the already initialized // mappings. This means that the same mappings will be applied to the test set. mlpack::data::Load("test_data.arff", trainingData, info); -\endcode +``` -When saving data, pass the same DatasetInfo object it was loaded with in order +When saving data, pass the same `DatasetInfo` object it was loaded with in order to unmap the categorical features correctly. The example below demonstrates this functionality: it loads the dataset, increments all non-categorical features by 1, and then saves the dataset with the same DatasetInfo it was loaded with. -\code +```c++ arma::mat dataset; // Load data into this matrix. mlpack::data::DatasetInfo info; // This will store the mappings. @@ -315,191 +287,208 @@ for (size_t i = 0; i < info.Dimensionality(); ++i) // Save the modified dataset using the same DatasetInfo. mlpack::data::Save("dataset-new.tsv", dataset, info); -\endcode +``` -There is more functionality to the DatasetInfo class; for more information, see -the mlpack::data::DatasetInfo documentation. +There is more functionality to the `DatasetInfo` class; for more information, +see the `mlpack::data::DatasetInfo` documentation. -@section intro_imagetut Loading and Saving Images +## Loading and Saving Images Image datasets are becoming increasingly popular in deep learning. -mlpack's image saving/loading functionality is based on [stb/](https://github.com/nothings/stb). +mlpack's image saving/loading functionality is based on +[STB](https://github.com/nothings/stb). -@section model_api_imagetut Image Utilities API +### Image Utilities API -Image utilities supports loading and saving of images. +mlpack's image utilities support loading and saving of images. -It supports filetypes "jpg", "png", "tga", "bmp", "psd", "gif", "hdr", "pic", -"pnm" for loading and "jpg", "png", "tga", "bmp", "hdr" for saving. +There is support for the following filetypes: `jpg`, `png`, `tga`, `bmp`, `psd`, +`gif`, `hdr`, `pic`, `pnm` for loading, and `jpg`, `png`, `tga`, `bmp`, `hdr` +for saving. The datatype associated is unsigned char to support RGB values in the range 1-255. To feed data into the network typecast of `arma::Mat` may be required. -Images are stored in the matrix as (width * height * channels, NumberOfImages). -Therefore @c imageMatrix.col(0) would be the first image if images are loaded in -@c imageMatrix. +Images are stored in the matrix as +`(width * height * channels, numberOfImages)`. Therefore `imageMatrix.col(0)` +would be the first image if images are loaded in `imageMatrix`. -@section imageinfo_api_imagetut Accessing Metadata of Images: ImageInfo +### Accessing Metadata of Images: ImageInfo -ImageInfo class contains the metadata of the images. -@code +`ImageInfo` class contains the metadata of the images. + +```c++ ImageInfo(const size_t width, const size_t height, const size_t channels, const size_t quality = 90); -@endcode +``` -The @c quality member denotes the compression of the image if it is saved as +The `quality` member denotes the compression of the image if it is saved as `jpg`; it takes values from 0 to 100. -@section load_api_imagetut Loading Images in C++ +### Loading Images in C++ -Standalone loading of images. +Standalone loading of images. Below is the signature of the +`mlpack::data::Load()` method for images: -@code +```c++ template bool Load(const std::string& filename, arma::Mat& matrix, ImageInfo& info, const bool fatal); -@endcode +``` The example below loads a test image. It also fills up the ImageInfo class object. -@code +```c++ data::ImageInfo info; data::Load("test_image.png", matrix, info, false); -@endcode +``` -ImageInfo requires height, width, number of channels of the image. +`ImageInfo` requires height, width, number of channels of the image. -@code +```c++ size_t height = 64, width = 64, channels = 1; data::ImageInfo info(width, height, channels); -@endcode +``` More than one image can be loaded into the same matrix. -Loading multiple images: +Loading multiple images: below is the signature of the `mlpack::data::Load()` +method to load multiple images. -@code +```c++ template bool Load(const std::vector& files, arma::Mat& matrix, ImageInfo& info, const bool fatal); -@endcode +``` -@code +Here is example usage: + +```c++ data::ImageInfo info; std::vector> files{"test_image1.bmp","test_image2.bmp"}; data::Load(files, matrix, info, false); -@endcode +``` -@section save_api_imagetut Saving Images in C++ +### Saving Images in C++ -Save images expects a matrix of type unsigned char in the form (width * height * channels, NumberOfImages). -Just like load it can be used to save one image or multiple images. Besides image data it also expects the shape of the image as input (width, height, channels). +`mlpack::data::Save()` images expects a matrix of type `unsigned char` in the +form `(width * height * channels, numberOfImages)`. Just like `Load()`, it can +be used to save one image or multiple images. Besides image data it also expects +the shape of the image as input `(width, height, channels)`. -Saving one image: +Saving one image: below is the signature of the `mlpack::data::Save()` method +for saving one image. -@code - template - bool Save(const std::string& filename, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal, - const bool transpose); -@endcode +```c++ +template +bool Save(const std::string& filename, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +``` -@code - data::ImageInfo info; - info.width = info.height = 25; - info.channels = 3; - info.quality = 90; - data::Save("test_image.bmp", matrix, info, false, true); -@endcode +Below is example usage: + +```c++ +data::ImageInfo info; +info.width = info.height = 25; +info.channels = 3; +info.quality = 90; +data::Save("test_image.bmp", matrix, info, false, true); +``` If the matrix contains more than one image, only the first one is saved. -Saving multiple images: +Saving multiple images: below is the signature of the `mlpack::data::Save()` +method for saving multiple images. -@code - template - bool Save(const std::vector& files, - arma::Mat& matrix, - ImageInfo& info, - const bool fatal, - const bool transpose); -@endcode +```c++ +template +bool Save(const std::vector& files, + arma::Mat& matrix, + ImageInfo& info, + const bool fatal, + const bool transpose); +``` -@code - data::ImageInfo info; - info.width = info.height = 25; - info.channels = 3; - info.quality = 90; - std::vector> files{"test_image1.bmp", "test_image2.bmp"}; - data::Save(files, matrix, info, false, true); -@endcode +Below is example usage: + +```c++ +data::ImageInfo info; +info.width = info.height = 25; +info.channels = 3; +info.quality = 90; +std::vector> files{"test_image1.bmp", "test_image2.bmp"}; +data::Save(files, matrix, info, false, true); +``` Multiple images are saved according to the vector of filenames specified. -@section formatmodels Loading and saving models +## Loading and Saving Models -Using \c cereal, mlpack is able to load and save machine learning +Using `cereal`, mlpack is able to load and save machine learning models with ease. These models can currently be saved in three formats: - - binary (.bin); this is not human-readable, but it is small - - json (.json); this is sort of human-readable and relatively small - - xml (.xml); this is human-readable but very verbose and large + - binary (`.bin`); this is not human-readable, but it is small + - json (`.json`); this is sort of human-readable and relatively small + - xml (`.xml`); this is human-readable but very verbose and large The type of file to save is determined by the given file extension, as with the other loading and saving functionality in mlpack. Below is an example where a -dataset stored as TSV and labels stored as ASCII text are used to train a -logistic regression model, which is then saved to model.xml. +dataset stored as TSV and labels stored as ASCII text are used with mlpack's +command line programs to train a logistic regression model, which is then saved +to `model.xml`. -\code +```sh $ mlpack_logistic_regression -t training_dataset.tsv -l training_labels.txt \ > -M model.xml -\endcode +``` Many mlpack command-line programs have support for loading and saving models -through the \c --input_model_file (\c -m) and \c --output_model_file (\c -M) +through the `--input_model_file` (`-m`) and `--output_model_file` (`-M`) options; for more information, see the documentation for each program -(accessible by passing \c --help as a parameter). +(accessible by passing `--help` as a parameter). -@section formatmodelscpp Loading and saving models in C++ +mlpack's bindings to other languages, similarly, have `input_model` parameters, +and depending on the language, may have `output_model` parameters (or may simply +return models as part of the output). -mlpack uses the \c cereal library internally to perform loading -and saving of models, and provides convenience overloads of mlpack::data::Load() -and mlpack::data::Save() to load and save these models. +### Loading and Saving Models in C++ + +mlpack uses the `cereal` library internally to perform loading and saving of +models, and provides convenience overloads of `mlpack::data::Load()` and +`mlpack::data::Save()` to load and save these models. To be serializable, a class must implement the method -\code +```c++ template void serialize(Archive& ar); -\endcode +``` -\note For more information on this method and how it works, see the -cereal documentation at https://uscilab.github.io/cereal/index.html. +[cereal documentation](https://uscilab.github.io/cereal/index.html). -\note -Examples of serialize() methods can be found in most classes; one fairly -straightforward example is found \ref mlpack::math::Range::serialize() -"in the mlpack::math::Range class". A more complex example is found -\ref mlpack::tree::BinarySpaceTree::serialize() "in the mlpack::tree::BinarySpaceTree class". +Examples of `serialize()` methods can be found in most classes; one fairly in +the `mlpack::math::Range` class. A more complex example is found in the +`mlpack::tree::BinarySpaceTree` class. -Using the mlpack::data::Load() and mlpack::data::Save() classes is easy if the -type being saved has a \c serialize() method implemented: simply call either +Using the `mlpack::data::Load()` and `mlpack::data::Save()` classes is easy if +the type being saved has a `serialize()` method implemented: simply call either function with a filename, a name for the object to save, and the object itself. -The example below, for instance, creates an mlpack::math::Range object and saves -it as range.txt. Then, that range is loaded from file into another -mlpack::math::Range object. +The example below, for instance, creates an `mlpack::math::Range` object and +saves it as `range.txt`. Then, that range is loaded from file into another +`mlpack::math::Range` object. -\code +```c++ // Create range and save it. mlpack::math::Range r(0.0, 5.0); mlpack::data::Save("range.json", "range", r); @@ -507,28 +496,26 @@ mlpack::data::Save("range.json", "range", r); // Load into new range. mlpack::math::Range newRange; mlpack::data::Load("range.json", "range", newRange); -\endcode +``` It is important to be sure that you load the appropriate type; if you save, for -instance, an mlpack::regression::LogisticRegression object and attempt to load -it as an mlpack::math::Range object, the load will fail and an exception will be -thrown. (When the object is saved as binary (.bin), it is possible that the +instance, an `mlpack::regression::LogisticRegression` object and attempt to load +it as an `mlpack::math::Range` object, the load will fail and an exception will +be thrown. (When the object is saved as binary (.bin), it is possible that the load will not fail, but instead load with mangled data, which is perhaps even worse!) -@section formatfinal Final notes +## Final Notes If the examples here are unclear, it would be worth looking into the ways that -mlpack::data::Load() and mlpack::data::Save() are used in the code. Some +`mlpack::data::Load()` and `mlpack::data::Save()` are used in the code. Some example files that may be useful to this end: - - src/mlpack/methods/logistic_regression/logistic_regression_main.cpp - - src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp - - src/mlpack/methods/neighbor_search/knn_main.cpp + - `src/mlpack/methods/logistic_regression/logistic_regression_main.cpp` + - `src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp` + - `src/mlpack/methods/neighbor_search/knn_main.cpp` If you are interested in adding support for more data types to mlpack, it would be preferable to add the support upstream to Armadillo instead, so that may be a better direction to go first. Then very little code modification for mlpack will be necessary. - -*/ diff --git a/doc/user/hpt.md b/doc/user/hpt.md new file mode 100644 index 0000000000..063a4fd86f --- /dev/null +++ b/doc/user/hpt.md @@ -0,0 +1,221 @@ +# Hyper-parameter Tuning + +mlpack implements a generic hyperparameter tuner that is able to tune both +continuous and discrete parameters of various different algorithms. This is an +important task---the performance of many machine learning algorithms can be +highly dependent on the hyperparameters that are chosen for that algorithm. +(One example: the choice of `k` for a `k`-nearest-neighbors classifier.) + +This hyper-parameter tuner is built on the same general concept as the +cross-validation classes (see the [cross-validation tutorial](cv.md)): given +some machine learning algorithm, some data, some performance measure, and a set +of hyperparameters, attempt to find the hyperparameter set that best optimizes +the performance measure on the given data with the given algorithm. + +mlpack's implementation of hyperparameter tuning is flexible, and is built in a +way that supports many algorithms and many optimizers. At the time of this +writing, complex hyperparameter optimization techniques are not available, but +the hyperparameter tuner does support these, should they be implemented in the +future. + +In this tutorial we will see the usage examples of the hyper-parameter tuning +module, and also more details about the `HyperParameterTuner` class. + +## Basic Usage + +The interface of the hyper-parameter tuning module is quite similar to the +interface of the [cross-validation module](cv.md). To construct a +`HyperParameterTuner` object you need to specify as template parameters what +machine learning algorithm, cross-validation strategy, performance measure, and +optimization strategy (`ens::GridSearch` will be used by default) you are going +to use. Then, you must pass the same arguments as for the cross-validation +classes: the data and labels (or responses) to use are given to the constructor, +and the possible hyperparameter values are given to the +`HyperParameterTuner::Optimize()` method, which returns the best algorithm +configuration as a `std::tuple<>`. + +Let's see some examples. + +Suppose we have the following data to train and validate on. + +```c++ +// 100-point 5-dimensional random dataset. +arma::mat data = arma::randu(5, 100); +// Noisy responses retrieved by a random linear transformation of data. +arma::rowvec responses = arma::randu(5) * data + + 0.1 * arma::randn(100); +``` + +Given the dataset above, we can use the following code to try to find a good +`lambda` value for `LinearRegression`. Here we use `cv::SimpleCV` instead of +k-fold cross-validation to save computation time. + +```c++ +// Using 80% of data for training and remaining 20% for assessing MSE. +double validationSize = 0.2; +HyperParameterTuner hpt(validationSize, + data, responses); + +// Finding a good value for lambda from the discrete set of values 0.0, 0.001, +// 0.01, 0.1, and 1.0. +arma::vec lambdas{0.0, 0.001, 0.01, 0.1, 1.0}; +double bestLambda; +std::tie(bestLambda) = hpt.Optimize(lambdas); +``` + +In this example we have used `ens::GridSearch` (the default optimizer) to find a +good value for the `lambda` hyper-parameter. For that we have specified what +values should be tried. + +## Fixed Arguments + +When some hyper-parameters should not be optimized, you can specify values for +them with the `Fixed()` method as in the following example of trying to find +good `lambda1` and `lambda2` values for `LARS` (least-angle regression). + +```c++ +HyperParameterTuner hpt2(validationSize, data, + responses); + +// The hyper-parameter tuner should not try to change the transposeData or +// useCholesky parameters. +bool transposeData = true; +bool useCholesky = false; + +// We wish only to search for the best lambda1 and lambda2 values. +arma::vec lambda1Set{0.0, 0.001, 0.01, 0.1, 1.0}; +arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0}; + +double bestLambda1, bestLambda2; +std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Fixed(transposeData), + Fixed(useCholesky), lambda1Set, lambda2Set); +``` + +Note that for the call to `hpt2.Optimize()`, we have used the same order of +arguments as they appear in the corresponding `LARS` constructor: + +```c++ +LARS(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData = true, + const bool useCholesky = false, + const double lambda1 = 0.0, + const double lambda2 = 0.0, + const double tolerance = 1e-16); +``` + +## Gradient-based optimization + +In some cases we may wish to optimize a hyperparameter over the space of all +possible real values, instead of providing a grid in which to search. +Alternately, we may know approximately optimal values from a grid search for +real-valued hyperparameters, but wish to further tune those values. + +In this case, we can use a gradient-based optimizer for hyperparameter search. +In the following example, we try to optimize the `lambda1` and `lambda2` +hyper-parameters for `LARS` with the `ens::GradientDescent` optimizer. + +```c++ +HyperParameterTuner hpt3(validationSize, + data, responses); + +// GradientDescent can be adjusted in the following way. +hpt3.Optimizer().StepSize() = 0.1; +hpt3.Optimizer().Tolerance() = 1e-15; + +// We can set up values used for calculating gradients. +hpt3.RelativeDelta() = 0.01; +hpt3.MinDelta() = 1e-10; + +double initialLambda1 = 0.001; +double initialLambda2 = 0.002; + +double bestGDLambda1, bestGDLambda2; +std::tie(bestGDLambda1, bestGDLambda2) = hpt3.Optimize(Fixed(transposeData), + Fixed(useCholesky), initialLambda1, initialLambda2); +``` + +## The `HyperParameterTuner` class + +The `HyperParameterTuner` class is very similar to the `KFoldCV` and `SimpleCV` +classes (see the [cross-validation tutorial](cv.md) for more information on +those two classes), but there are a few important differences. + +First, the `HyperParameterTuner` accepts five different hyperparameters; only +the first three of these are required: + + - `MLAlgorithm` This is the algorithm to be used. + - `Metric` This is the performance measure to be used; see + [the cross-validation tutorial](cv.md) for more information. + - `CVType` This is the type of cross-validation to be used for evaluating the + performance measure; this should be `KFoldCV` or `SimpleCV`. + - `OptimizerType` This is the type of optimizer to use; it can be + `GridSearch` or a gradient-based optimizer. + - `MatType` This is the type of data matrix to use. The default is + `arma::mat`. This only needs to be changed if you are specifically + using sparse data, or if you want to use a numeric type other than + `double`. + +The last two template parameters are automatically inferred by the +`HyperParameterTuner` and should not need to be manually specified, unless an +unconventional data type like `arma::fmat` is being used for data points. + +Typically, `SimpleCV` is a good choice for `CVType` because it takes so much +less time to compute than full `KFoldCV`; however, the disadvantage is that +`SimpleCV` might give a somewhat more noisy estimate of the performance measure +on unseen test data. + +The constructor for the `HyperParameterTuner` is called with exactly the same +arguments as the corresponding `CVType` that has been chosen. For more +information on that, please see the [cross-validation tutorial](cv.md). As an +example, if we are using `SimpleCV` and wish to hold out 20% of the dataset as a +validation set, we might construct a `HyperParameterTuner` like this: + +```c++ +// We will use LinearRegression as the MLAlgorithm, and MSE as the performance +// measure. Our dataset is 'dataset' and the responses are 'responses'. +HyperParameterTuner hpt(0.2, dataset, + responses); +``` + +Next, we must set up the hyperparameters to be optimized. If we are doing a +grid search with the \c ens::GridSearch optimizer (the +default), then we only need to pass a `std::vector` (for non-numeric +hyperparameters) or an `arma::vec` (for numeric hyperparameters) containing all +of the possible choices that we wish to search over. + +For instance, a set of numeric values might be chosen like this, for the +`lambda` parameter (of type `double`): + +```c++ +arma::vec lambdaSet = arma::vec("0.0 0.1 0.5 1.0"); +``` + +Similarly, a set of non-numeric values might be chosen like this, for the +`intercept` parameter: + +```c++ +std::vector interceptSet = { false, true }; +``` + +Once all of these are set up, the `HyperParameterTuner::Optimize()` method may +be called to find the best set of hyperparameters: + +```c++ +bool intercept; +double lambda; +std::tie(lambda, intercept) = hpt.Optimize(lambdaSet, interceptSet); +``` + +Alternately, the `Fixed()` method (detailed in the "Fixed arguments" section) +can be used to fix the values of some parameters. + +For continuous optimizers like `ens::GradientDescent`, a range does not need to +be specified but instead only a single value. See the "Gradient-Based +Optimization" section for more details. + +## Further documentation + +For more information on the `HyperParameterTuner` class, see the source code fro +the `mlpack::hpt::HyperParameterTuner` class (it is very well commented!), and +the [cross-validation tutorial](cv.md). diff --git a/doc/guide/matrices.hpp b/doc/user/matrices.md similarity index 70% rename from doc/guide/matrices.hpp rename to doc/user/matrices.md index 73ba2d95fb..d2159f77d5 100644 --- a/doc/guide/matrices.hpp +++ b/doc/user/matrices.md @@ -1,18 +1,15 @@ -/*! @page matrices Matrices in mlpack - -@section matintro Introduction +# Matrices in mlpack mlpack uses Armadillo matrices for matrix support. Armadillo is a fast C++ matrix library which makes use of advanced template techniques to provide the fastest possible matrix operations. -Documentation on Armadillo can be found on their website: - -http://arma.sourceforge.net/docs.html +Documentation on Armadillo can be found on [the Armadillo +website](http://arma.sourceforge.net/docs.html). Nonetheless, there are a few further caveats for mlpack Armadillo usage. -@section format Column-major Matrices +## Column-major matrices Armadillo matrices are stored in a column-major format; this means that on disk, each column is located in contiguous memory. @@ -24,32 +21,32 @@ most standard machine learning texts! Major implications of this are for linear algebra. For instance, the covariance of a matrix is typically -@f[ +``` C = X^T X -@f] +``` but for a column-wise matrix, it is -@f[ +``` C = X X^T -@f] +``` and this is very important to keep in mind! If your mlpack code is not working, this may be a factor in why. -@section loading Loading Matrices +## Loading matrices -mlpack provides a data::Load() and data::Save() function, which should be used -instead of Armadillo's loading and saving functions. +mlpack provides a `data::Load()` and `data::Save()` function, which should be +used instead of Armadillo's loading and saving functions. Most machine learning data is stored in row-major format; a CSV, for example, will generally have one observation per line and each column will correspond to a dimension. -The data::Load() and data::Save() functions transpose the matrix upon loading, -meaning that the following CSV: +The `data::Load()` and `data::Save()` functions transpose the matrix upon +loading, meaning that the following CSV: -@code +```sh $ cat data.csv 3,3,3,3,0 3,4,4,3,0 @@ -64,12 +61,8 @@ $ cat data.csv 3,3,4,2,0 3,6,4,2,0 2,4,4,2,0 -@endcode +``` is actually loaded with 5 rows and 13 columns, not 13 rows and 5 columns like the CSV is written. More information on mlpack's loading functionality can be -found in \ref formatdoc. - -This is important to remember! - -*/ +found in [the formats tutorial](formats.md). diff --git a/doc/user/sample_ml_app.md b/doc/user/sample_ml_app.md new file mode 100644 index 0000000000..0bb3df5397 --- /dev/null +++ b/doc/user/sample_ml_app.md @@ -0,0 +1,203 @@ +# Sample C++ ML App for Windows + +*by German Lancioni* + +This tutorial will help you create a sample machine learning app using +mlpack/C++. Although this app does not cover all the mlpack capabilities, it +will walkthrough several APIs to understand how everything connects. This +Windows sample app is created using Visual Studio, but you can easily adapt it +to a different platform by following the provided source code. + +*Note*: before starting, make sure you have built mlpack for Windows following +this [Windows guide](build_windows.md). + +## Creating the Visual Studio project + +- Open Visual Studio and create a new project (Windows Console Application) +- For this sample, the project is named `sample-ml-app` + +## Project configuration + +There are different ways in which you can configure your project to link with +dependencies. This configuration is for x64 Debug Mode. If you need Release +Mode, please change the paths accordingly (assuming you have built mlpack and +dependencies in Release Mode). + +- Right click on the project and select Properties, select the x64 Debug profile +- Under C/C++ > General > Additional Include Directories add: +``` + - C:\mlpack\armadillo-9.800.3\include + - C:\mlpack\mlpack-3.4.2\src +``` +- Under Build Events > Post-Build Event > Command Line add: +``` + - xcopy /y "C:\mlpack\mlpack-3.4.2\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) +``` + +*Note*: recent versions of Visual Studio set "Conformance Mode" enabled by +default. This causes some issues with the Armadillo library. If you encounter +this issue, disable "Conformance Mode" under C/C++ > Language. + +## The App's Goal + +This app aims to exercise an end-to-end machine learning workflow. We will +cover: + +- Loading and preparing a dataset +- Training (using Random Forest as example) +- Computing the training accuracy +- Cross-Validation using K-Fold +- Metrics gathering (accuracy, precision, recall, F1) +- Saving the trained model to disk +- Loading the model +- Classifying a new sample + +## Headers and namespaces + +For this app, we will need to include the mlpack header (i.e. add into +`stdafx.h`): + +```c++ +#include "mlpack.hpp" +``` + +Also, we will use the following namespaces: + +```c++ +using namespace arma; +using namespace mlpack; +using namespace mlpack::tree; +using namespace mlpack::cv; +``` + +## Loading the dataset + +The first step is about loading the dataset. Different dataset file formats are +supported, but here we load a CSV dataset, and we assume the labels don't +require normalization. + +*Note*: make sure you update the path to your dataset file. For this sample, you +can simply copy `mlpack/tests/data/german.csv` and paste into a new `data` +folder in your project directory. + +```c++ +mat dataset; +bool loaded = mlpack::data::Load("data/german.csv", dataset); +if (!loaded) + return -1; +``` + +Then we need to extract the labels from the last dimension of the dataset and +remove the labels from the training set: + +```c++ +Row labels; +labels = conv_to>::from(dataset.row(dataset.n_rows - 1)); +dataset.shed_row(dataset.n_rows - 1); +``` + +We now have our dataset ready for training. + +## Training + +This app will use a Random Forest classifier. At first we define the classifier +parameters and then we create the classifier to train it. + +```c++ +const size_t numClasses = 2; +const size_t minimumLeafSize = 5; +const size_t numTrees = 10; + +RandomForest rf; + +rf = RandomForest(dataset, labels, + numClasses, numTrees, minimumLeafSize); +``` + +Now that the training is completed, we quickly compute the training accuracy: + +```c++ +Row predictions; +rf.Classify(dataset, predictions); +const size_t correct = arma::accu(predictions == labels); +cout << "\nTraining Accuracy: " << (double(correct) / double(labels.n_elem)); +``` + +## Cross-validating + +Instead of training the Random Forest directly, we could also use K-fold +cross-validation for training, which will give us a measure of performance on a +held-out test set. This can give us a better estimate of how the model will +perform when given new data. We also define which metric to use in order to +assess the quality of the trained model. + +```c++ +const size_t k = 10; +KFoldCV, Accuracy> cv(k, + dataset, labels, numClasses); +double cvAcc = cv.Evaluate(numTrees, minimumLeafSize); +cout << "\nKFoldCV Accuracy: " << cvAcc; +``` + +To compute other relevant metrics, such as Precision, Recall and F1: + +```c++ +double cvPrecision = Precision::Evaluate(rf, dataset, labels); +cout << "\nPrecision: " << cvPrecision; + +double cvRecall = Recall::Evaluate(rf, dataset, labels); +cout << "\nRecall: " << cvRecall; + +double cvF1 = F1::Evaluate(rf, dataset, labels); +cout << "\nF1: " << cvF1; +``` + +## Saving the model + +Now that our model is trained and validated, we save it to a file so we can use +it later. Here we save the model that was trained using the entire dataset. +Alternatively, we could extract the model from the cross-validation stage by +using `cv.Model()`. + +```c++ +mlpack::data::Save("mymodel.xml", "model", rf, false); +``` + +We can also save the model in `bin` format (`"mymodel.bin"`) which would result +in a smaller file. + +## Loading the model + +In a real-life application, you may want to load a previously trained model to +classify new samples. We load the model from a file using: + +```c++ +mlpack::data::Load("mymodel.xml", "model", rf); +``` + +## Classifying a new sample + +Finally, the ultimate goal is to classify a new sample using the previously +trained model. Since the Random Forest classifier provides both predictions and +probabilities, we obtain both. + +```c++ +// Create a test sample containing only one point. Because Armadillo is +// column-major, this matrix has one column (one point) and the number of rows +// is equal to the dimensionality of the point (23). +mat sample("2; 12; 2; 13; 1; 2; 2; 1; 3; 24; 3; 1; 1; 1; 1; 1; 0; 1; 0; 1;" + " 0; 0; 0"); +mat probabilities; +rf.Classify(sample, predictions, probabilities); +u64 result = predictions.at(0); +cout << "\nClassification result: " << result << " , Probabilities: " << + probabilities.at(0) << "/" << probabilities.at(1); +``` + +## Final thoughts + +Building real-life applications and services using machine learning can be +challenging. Hopefully, this tutorial provides a good starting point that covers +the basic workflow you may need to follow while developing it. You can take a +look at the entire source code in the provided sample project located here: +`doc/examples/sample-ml-app`. diff --git a/src/mlpack.hpp b/src/mlpack.hpp new file mode 100644 index 0000000000..e4f05d54b5 --- /dev/null +++ b/src/mlpack.hpp @@ -0,0 +1,70 @@ +/** + * @file mlpack.hpp + * + * Include all of mlpack! When this file is included, all components of mlpack + * are available. + * + * Note that by default, serialization for ANN layers is not enabled, since this + * will cause the build time to be very long. If you plan to serialize a neural + * network, simply include mlpack like this: + * + * ``` + * #define MLPACK_ENABLE_ANN_SERIALIZATION + * #include + * ``` + */ +#ifndef MLPACK_HPP +#define MLPACK_HPP + +// Include all of the core library components. +#include "mlpack/base.hpp" +#include "mlpack/prereqs.hpp" +#include "mlpack/core.hpp" + +// Now include all of the methods. +#include "mlpack/methods/adaboost.hpp" +#include "mlpack/methods/amf.hpp" +#include "mlpack/methods/ann.hpp" +#include "mlpack/methods/approx_kfn.hpp" +#include "mlpack/methods/bayesian_linear_regression.hpp" +#include "mlpack/methods/bias_svd.hpp" +#include "mlpack/methods/block_krylov_svd.hpp" +#include "mlpack/methods/cf.hpp" +#include "mlpack/methods/dbscan.hpp" +#include "mlpack/methods/decision_tree.hpp" +#include "mlpack/methods/det.hpp" +#include "mlpack/methods/emst.hpp" +#include "mlpack/methods/fastmks.hpp" +#include "mlpack/methods/gmm.hpp" +#include "mlpack/methods/hmm.hpp" +#include "mlpack/methods/hoeffding_trees.hpp" +#include "mlpack/methods/kde.hpp" +#include "mlpack/methods/kernel_pca.hpp" +#include "mlpack/methods/kmeans.hpp" +#include "mlpack/methods/lars.hpp" +#include "mlpack/methods/linear_regression.hpp" +#include "mlpack/methods/lmnn.hpp" +#include "mlpack/methods/local_coordinate_coding.hpp" +#include "mlpack/methods/logistic_regression.hpp" +#include "mlpack/methods/lsh.hpp" +#include "mlpack/methods/matrix_completion.hpp" +#include "mlpack/methods/mean_shift.hpp" +#include "mlpack/methods/naive_bayes.hpp" +#include "mlpack/methods/nca.hpp" +#include "mlpack/methods/neighbor_search.hpp" +#include "mlpack/methods/pca.hpp" +#include "mlpack/methods/perceptron.hpp" +#include "mlpack/methods/quic_svd.hpp" +#include "mlpack/methods/radical.hpp" +#include "mlpack/methods/random_forest.hpp" +#include "mlpack/methods/randomized_svd.hpp" +#include "mlpack/methods/range_search.hpp" +#include "mlpack/methods/rann.hpp" +#include "mlpack/methods/regularized_svd.hpp" +#include "mlpack/methods/reinforcement_learning.hpp" +#include "mlpack/methods/softmax_regression.hpp" +#include "mlpack/methods/sparse_autoencoder.hpp" +#include "mlpack/methods/sparse_coding.hpp" +#include "mlpack/methods/svdplusplus.hpp" + +#endif diff --git a/src/mlpack/CMakeLists.txt b/src/mlpack/CMakeLists.txt index fa5f1e176a..96c69386d7 100644 --- a/src/mlpack/CMakeLists.txt +++ b/src/mlpack/CMakeLists.txt @@ -10,6 +10,9 @@ endif () # At install time, we simply install the src/ directory to include/ (though we # omit bindings/ and tests/). +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/../mlpack.hpp" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/base.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/core.hpp" diff --git a/src/mlpack/bindings/markdown/print_docs.cpp b/src/mlpack/bindings/markdown/print_docs.cpp index 83d5724ac0..38cee5aeeb 100644 --- a/src/mlpack/bindings/markdown/print_docs.cpp +++ b/src/mlpack/bindings/markdown/print_docs.cpp @@ -19,8 +19,12 @@ #include "replace_all_copy.hpp" // Make sure that this is defined. -#ifndef DOXYGEN_PREFIX -#define DOXYGEN_PREFIX "https://mlpack.org/doc/mlpack-git/doxygen/" +#ifndef SRC_PREFIX +#define SRC_PREFIX "https://github.com/mlpack/mlpack/blob/master/src/" +#endif + +#ifndef DOC_PREFIX +#define DOC_PREFIX "https://github.com/mlpack/mlpack/blob/master/doc/" #endif using namespace std; @@ -264,10 +268,14 @@ void PrintDocs(const string& bindingName, cout << doc.seeAlso[j].first; cout << "]("; - // We need special handling of Doxygen information. - if (doc.seeAlso[j].second.substr(0, 8) == "@doxygen") + // We need special handling of source links. + if (doc.seeAlso[j].second.substr(0, 4) == "@src") { - cout << DOXYGEN_PREFIX << doc.seeAlso[j].second.substr(9); + cout << SRC_PREFIX << doc.seeAlso[j].second.substr(5); + } + else if (doc.seeAlso[j].second.substr(0, 4) == "@doc") + { + cout << DOC_PREFIX << doc.seeAlso[j].second.substr(5); } else if (doc.seeAlso[j].second[0] == '#') { diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index 39749ec824..44d3eefce3 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -1,8 +1,7 @@ /** * @file core.hpp * - * Include all of the base components required to write mlpack methods, and the - * main mlpack Doxygen documentation. + * Include all of the base components required to write mlpack methods. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -13,18 +12,12 @@ #define MLPACK_CORE_HPP /** - * @mainpage mlpack Documentation - * - * @section intro_sec Introduction - * * mlpack is an intuitive, fast, and flexible C++ machine learning library with * bindings to other languages. It is meant to be a machine learning analog to * LAPACK, and aims to implement a wide array of machine learning methods and * function as a "swiss army knife" for machine learning researchers. The * mlpack website can be found at https://mlpack.org. * - * @section howto How To Use This Documentation - * * This documentation is API documentation similar to Javadoc. It isn't * necessarily a tutorial, but it does provide detailed documentation on every * namespace, method, and class. @@ -33,31 +26,6 @@ * browsing the list of namespaces provides some insight as to the breadth of * the methods contained in the library. * - * To generate this documentation in your own local copy of mlpack, you can use - * the 'doc' CMake target, which is available if CMake has found Doxygen, from - * the build directory: - * - * @code - * $ make doc - * @endcode - * - * @section tutorial Tutorials - * - * A few short tutorials on how to use mlpack are given below. - * - * - @ref build - * - @ref build_windows - * - @ref matrices - * - @ref iodoc - * - @ref timer - * - @ref sample - * - @ref sample_ml_app - * - @ref cv - * - @ref hpt_guide - * - @ref verinfo - * - * @section remarks Final Remarks - * * For the list of contributors to mlpack, see * https://www.mlpack.org/community.html. This library would not be possible * without everyone's hard work and contributions! diff --git a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp index 73a532b03e..70e3aa264b 100644 --- a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp @@ -149,7 +149,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) for (size_t i = lastDim + 1; i < axes.size(); ++i) { for (size_t j = 0; j < tree->NumChildren(); ++j) - axes[i] = axes[i] & + axes[i] = axes[i] && tree->Child(j).AuxiliaryInfo().SplitHistory().history[i]; if (axes[i] == true) { @@ -164,7 +164,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) { axes[i] = true; for (size_t j = 0; j < tree->NumChildren(); ++j) - axes[i] = axes[i] & + axes[i] = axes[i] && tree->Child(j).AuxiliaryInfo().SplitHistory().history[i]; if (axes[i] == true) { diff --git a/src/mlpack/core/util/param.hpp b/src/mlpack/core/util/param.hpp index 6ca2a8ed6c..5879476b59 100644 --- a/src/mlpack/core/util/param.hpp +++ b/src/mlpack/core/util/param.hpp @@ -178,8 +178,10 @@ * - A direct URL, starting with http:// or https://. * - A page anchor for documentation, referencing another binding by its CMake * binding name, i.e. "#knn". - * - A link to a Doxygen page, using the mangled Doxygen name after a - * '\@doxygen/', i.e., "@doxygen/mlpack1_1_adaboost1_1_AdaBoost". + * - A link to a source file, using the source path after '@src', i.e., + * "@src/mlpack/core/util/param.hpp" + * - A link to a documentation file, using the path after '@doc', i.e., + * "@doc/user/matrices.md" */ #ifdef __COUNTER__ #define BINDING_SEE_ALSO(DESCRIPTION, LINK) static \ diff --git a/src/mlpack/methods/adaboost/adaboost_main.cpp b/src/mlpack/methods/adaboost/adaboost_main.cpp index f88d0ce103..deb7169801 100644 --- a/src/mlpack/methods/adaboost/adaboost_main.cpp +++ b/src/mlpack/methods/adaboost/adaboost_main.cpp @@ -122,7 +122,7 @@ BINDING_SEE_ALSO("Improved boosting algorithms using confidence-rated " BINDING_SEE_ALSO("Perceptron", "#perceptron"); BINDING_SEE_ALSO("Decision Stump", "#decision_stump"); BINDING_SEE_ALSO("mlpack::adaboost::AdaBoost C++ class documentation", - "@doxygen/classmlpack_1_1adaboost_1_1AdaBoost.html"); + "@src/mlpack/methods/adaboost/adaboost.hpp"); // Input for training. PARAM_MATRIX_IN("training", "Dataset for training AdaBoost.", "t"); diff --git a/src/mlpack/methods/adaboost/adaboost_train_main.cpp b/src/mlpack/methods/adaboost/adaboost_train_main.cpp index 5975be7b7e..21328aaae1 100644 --- a/src/mlpack/methods/adaboost/adaboost_train_main.cpp +++ b/src/mlpack/methods/adaboost/adaboost_train_main.cpp @@ -91,7 +91,7 @@ BINDING_SEE_ALSO("Improved boosting algorithms using confidence-rated " BINDING_SEE_ALSO("Perceptron", "#perceptron"); BINDING_SEE_ALSO("Decision Stump", "#decision_stump"); BINDING_SEE_ALSO("mlpack::adaboost::AdaBoost C++ class documentation", - "@doxygen/classmlpack_1_1adaboost_1_1AdaBoost.html"); + "@src/mlpack/methods/adaboost/adaboost.hpp"); // Input for training. PARAM_MATRIX_IN_REQ("training", "Dataset for training AdaBoost.", "t"); diff --git a/src/mlpack/methods/ann.hpp b/src/mlpack/methods/ann.hpp index 4ae2acd4bb..2ee4f9f53f 100644 --- a/src/mlpack/methods/ann.hpp +++ b/src/mlpack/methods/ann.hpp @@ -1,7 +1,10 @@ /** * @file ann.hpp * - * Convenience include for mlpack/methods/ann/ann.hpp + * Convenience include for mlpack/methods/ann/ann.hpp. + * + * Note that serialization for neural networks is not enabled unless the + * MLPACK_ENABLE_ANN_SERIALIZATION macro is defined! */ #ifndef MLPACK_ANN_HPP #define MLPACK_ANN_HPP diff --git a/src/mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp b/src/mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp index a1f68dae2f..d82cf2d93a 100644 --- a/src/mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp @@ -56,7 +56,8 @@ class HardSigmoidFunction { y.set_size(size(x)); - for (size_t i = 0; i < x.n_elem; ++i) + #pragma omp for + for (size_t i = 0; i < (size_t) x.n_elem; ++i) y(i) = Fn(x(i)); } @@ -86,7 +87,8 @@ class HardSigmoidFunction { x.set_size(size(y)); - for (size_t i = 0; i < y.n_elem; ++i) + #pragma omp for + for (size_t i = 0; i < (size_t) y.n_elem; ++i) { x(i) = Deriv(y(i)); } diff --git a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp index d387e86474..c940627c00 100644 --- a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp @@ -74,7 +74,8 @@ class HardSwishFunction { y.set_size(size(x)); - for (size_t i = 0; i < x.n_elem; i++) + #pragma omp for + for (size_t i = 0; i < (size_t) x.n_elem; i++) y(i) = Fn(x(i)); } @@ -105,7 +106,8 @@ class HardSwishFunction { x.set_size(size(y)); - for (size_t i = 0; i < y.n_elem; i++) + #pragma omp for + for (size_t i = 0; i < (size_t) y.n_elem; i++) x(i) = Deriv(y(i)); } }; // class HardSwishFunction diff --git a/src/mlpack/methods/ann/ann.hpp b/src/mlpack/methods/ann/ann.hpp index cc4cc22492..c3ace77e9b 100644 --- a/src/mlpack/methods/ann/ann.hpp +++ b/src/mlpack/methods/ann/ann.hpp @@ -4,6 +4,9 @@ * * Convenience include for all aspects of the neural network framework in * mlpack. + * + * Note that serialization for neural networks is not enabled unless the + * MLPACK_ENABLE_ANN_SERIALIZATION macro is defined! */ #ifndef MLPACK_METHODS_ANN_ANN_HPP #define MLPACK_METHODS_ANN_ANN_HPP diff --git a/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp index 8497ff11d8..f968af4d9b 100644 --- a/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp @@ -37,7 +37,7 @@ template class FFTConvolution { public: - /* + /** * Perform a convolution through fft (valid mode). This method only supports * input which is even on the last dimension. In case of an odd input width, a * user can manually pad the input or specify the padLastDim parameter which @@ -73,7 +73,7 @@ class FFTConvolution input.n_rows - 1, input.n_cols - 1); } - /* + /** * Perform a convolution through fft (full mode). This method only supports * input which is even on the last dimension. In case of an odd input width, a * user can manually pad the input or specify the padLastDim parameter which @@ -120,7 +120,7 @@ class FFTConvolution 2 * (filter.n_cols - 1) + input.n_cols - 1); } - /* + /** * Perform a convolution through fft using 3rd order tensors. This method only * supports input which is even on the last dimension. In case of an odd input * width, a user can manually pad the input or specify the padLastDim @@ -151,7 +151,7 @@ class FFTConvolution } } - /* + /** * Perform a convolution through fft using dense matrix as input and a 3rd * order tensors as filter and output. This method only supports input which * is even on the last dimension. In case of an odd input width, a user can @@ -182,7 +182,7 @@ class FFTConvolution } } - /* + /** * Perform a convolution using a 3rd order tensors as input and output and a * dense matrix as filter. * diff --git a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp index c01fd6d749..1ab4fbb03c 100644 --- a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp @@ -35,7 +35,7 @@ template class NaiveConvolution { public: - /* + /** * Perform a convolution (valid mode). * * @param input Input used to perform the convolution. @@ -45,6 +45,8 @@ class NaiveConvolution * @param dH Stride of filter application in the y direction. * @param dilationW The dilation factor in x direction. * @param dilationH The dilation factor in y direction. + * @param appending If true, it will not initialize the output. Instead, + * it will append the results to the output. */ template static typename std::enable_if< @@ -55,17 +57,21 @@ class NaiveConvolution const size_t dW = 1, const size_t dH = 1, const size_t dilationW = 1, - const size_t dilationH = 1) + const size_t dilationH = 1, + const bool appending = false) { // Compute the output size. The filterRows and filterCols computation must // take into account the fact that dilation only adds rows or columns // *between* filter elements. So, e.g., a dilation of 2 on a kernel size of // 3x3 means an effective kernel size of 5x5, *not* 6x6. - const size_t filterRows = filter.n_rows * dilationH - (dilationH - 1); - const size_t filterCols = filter.n_cols * dilationW - (dilationW - 1); - const size_t outputRows = (input.n_rows - filterRows + dH) / dH; - const size_t outputCols = (input.n_cols - filterCols + dW) / dW; - output.zeros(outputRows, outputCols); + if (!appending) + { + const size_t filterRows = filter.n_rows * dilationH - (dilationH - 1); + const size_t filterCols = filter.n_cols * dilationW - (dilationW - 1); + const size_t outputRows = (input.n_rows - filterRows + dH) / dH; + const size_t outputCols = (input.n_cols - filterCols + dW) / dW; + output.zeros(outputRows, outputCols); + } // It seems to be about 3.5 times faster to use pointers instead of // filter(ki, kj) * input(leftInput + ki, topInput + kj) and output(i, j). @@ -87,7 +93,7 @@ class NaiveConvolution } } - /* + /** * Perform a convolution (full mode). * * @param input Input used to perform the convolution. @@ -97,6 +103,8 @@ class NaiveConvolution * @param dH Stride of filter application in the y direction. * @param dilationW The dilation factor in x direction. * @param dilationH The dilation factor in y direction. + * @param appending If true, it will not initialize the output. Instead, + * it will append the results to the output. */ template static typename std::enable_if< @@ -107,7 +115,8 @@ class NaiveConvolution const size_t dW = 1, const size_t dH = 1, const size_t dilationW = 1, - const size_t dilationH = 1) + const size_t dilationH = 1, + const bool appending = false) { // First, compute the necessary padding for the full convolution. It is // possible that this might be an overestimate. Note that these variables @@ -124,10 +133,10 @@ class NaiveConvolution paddingCols + input.n_cols - 1) = input; NaiveConvolution::Convolution(inputPadded, filter, - output, dW, dH, dilationW, dilationH); + output, dW, dH, dilationW, dilationH, appending); } - /* + /** * Perform a convolution using 3rd order tensors. * * @param input Input used to perform the convolution. @@ -137,6 +146,8 @@ class NaiveConvolution * @param dH Stride of filter application in the y direction. * @param dilationW The dilation factor in x direction. * @param dilationH The dilation factor in y direction. + * @param appending If true, it will not initialize the output. Instead, + * it will append the results to the output. */ template static void Convolution(const arma::Cube& input, @@ -145,24 +156,26 @@ class NaiveConvolution const size_t dW = 1, const size_t dH = 1, const size_t dilationW = 1, - const size_t dilationH = 1) + const size_t dilationH = 1, + const bool appending = false) { arma::Mat convOutput; NaiveConvolution::Convolution(input.slice(0), filter.slice(0), - convOutput, dW, dH, dilationW, dilationH); + convOutput, dW, dH, dilationW, dilationH, appending); - output = arma::Cube(convOutput.n_rows, convOutput.n_cols, - input.n_slices); + if (!appending) + output = arma::Cube(convOutput.n_rows, convOutput.n_cols, + input.n_slices); output.slice(0) = convOutput; for (size_t i = 1; i < input.n_slices; ++i) { NaiveConvolution::Convolution(input.slice(i), filter.slice(i), - output.slice(i), dW, dH, dilationW, dilationH); + output.slice(i), dW, dH, dilationW, dilationH, appending); } } - /* + /** * Perform a convolution using dense matrix as input and a 3rd order tensors * as filter and output. * @@ -173,6 +186,8 @@ class NaiveConvolution * @param dH Stride of filter application in the y direction. * @param dilationW The dilation factor in x direction. * @param dilationH The dilation factor in y direction. + * @param appending If true, it will not initialize the output. Instead, + * it will append the results to the output. */ template static void Convolution(const arma::Mat& input, @@ -181,24 +196,26 @@ class NaiveConvolution const size_t dW = 1, const size_t dH = 1, const size_t dilationW = 1, - const size_t dilationH = 1) + const size_t dilationH = 1, + const bool appending = false) { arma::Mat convOutput; NaiveConvolution::Convolution(input, filter.slice(0), - convOutput, dW, dH, dilationW, dilationH); + convOutput, dW, dH, dilationW, dilationH, appending); - output = arma::Cube(convOutput.n_rows, convOutput.n_cols, - filter.n_slices); + if (!appending) + output = arma::Cube(convOutput.n_rows, convOutput.n_cols, + filter.n_slices); output.slice(0) = convOutput; for (size_t i = 1; i < filter.n_slices; ++i) { NaiveConvolution::Convolution(input, filter.slice(i), - output.slice(i), dW, dH, dilationW, dilationH); + output.slice(i), dW, dH, dilationW, dilationH, appending); } } - /* + /** * Perform a convolution using a 3rd order tensors as input and output and a * dense matrix as filter. * @@ -208,7 +225,9 @@ class NaiveConvolution * @param dW Stride of filter application in the x direction. * @param dH Stride of filter application in the y direction. * @param dilationW The dilation factor in x direction. - * @param dilationH The dilation factor in y direction. + * @param dilationH The dilation factor in y direction.x + * @param appending If true, it will not initialize the output. Instead, + * it will append the results to the output. */ template static void Convolution(const arma::Cube& input, @@ -217,20 +236,22 @@ class NaiveConvolution const size_t dW = 1, const size_t dH = 1, const size_t dilationW = 1, - const size_t dilationH = 1) + const size_t dilationH = 1, + const bool appending = false) { arma::Mat convOutput; NaiveConvolution::Convolution(input.slice(0), filter, - convOutput, dW, dH, dilationW, dilationH); + convOutput, dW, dH, dilationW, dilationH, appending); - output = arma::Cube(convOutput.n_rows, convOutput.n_cols, - input.n_slices); + if (!appending) + output = arma::Cube(convOutput.n_rows, convOutput.n_cols, + input.n_slices); output.slice(0) = convOutput; for (size_t i = 1; i < input.n_slices; ++i) { NaiveConvolution::Convolution(input.slice(i), filter, - output.slice(i), dW, dH, dilationW, dilationH); + output.slice(i), dW, dH, dilationW, dilationH, appending); } } }; // class NaiveConvolution diff --git a/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp index ce9a021127..35ee960055 100644 --- a/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp @@ -38,7 +38,7 @@ template class SVDConvolution { public: - /* + /** * Perform a convolution (valid or full mode) using singular value * decomposition. By using singular value decomposition of the filter matrix * the convolution can be expressed as a sum of outer products. Each product @@ -49,8 +49,6 @@ class SVDConvolution * @param input Input used to perform the convolution. * @param filter Filter used to perform the conolution. * @param output Output data that contains the results of the convolution. - * @param dW Stride of filter application in the x direction. - * @param dH Stride of filter application in the y direction. */ template static void Convolution(const arma::Mat& input, @@ -109,14 +107,12 @@ class SVDConvolution } } - /* + /** * Perform a convolution using 3rd order tensors. * * @param input Input used to perform the convolution. * @param filter Filter used to perform the conolution. * @param output Output data that contains the results of the convolution. - * @param dW Stride of filter application in the x direction. - * @param dH Stride of filter application in the y direction. */ template static void Convolution(const arma::Cube& input, @@ -138,15 +134,13 @@ class SVDConvolution } } - /* + /** * Perform a convolution using dense matrix as input and a 3rd order tensors * as filter and output. * * @param input Input used to perform the convolution. * @param filter Filter used to perform the conolution. * @param output Output data that contains the results of the convolution. - * @param dW Stride of filter application in the x direction. - * @param dH Stride of filter application in the y direction. */ template static void Convolution(const arma::Mat& input, @@ -167,15 +161,13 @@ class SVDConvolution } } - /* + /** * Perform a convolution using a 3rd order tensors as input and output and a * dense matrix as filter. * * @param input Input used to perform the convolution. * @param filter Filter used to perform the conolution. * @param output Output data that contains the results of the convolution. - * @param dW Stride of filter application in the x direction. - * @param dH Stride of filter application in the y direction. */ template static void Convolution(const arma::Cube& input, diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index 08a9c7d641..ecf6c9e592 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -377,6 +377,9 @@ class ConvolutionType : public Layer //! Locally-stored padding layer. ann::Padding padding; + //! Locally-stored padding layer for backward pass. + ann::Padding paddingBackward; + //! Type of padding. std::string paddingType; @@ -384,6 +387,12 @@ class ConvolutionType : public Layer size_t inMaps; //! Locally-cached higher-order input dimensions. size_t higherInDimensions; + + //! Locally-stored apparent width. + size_t apparentWidth; + + //! Locally-stored apparent height. + size_t apparentHeight; }; // class Convolution // Standard Convolution layer. diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index e866a0e4ee..923fca48c0 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -130,9 +130,12 @@ ConvolutionType< padHTop(other.padHTop), useBias(other.useBias), padding(other.padding), + paddingBackward(other.paddingBackward), paddingType(other.paddingType), inMaps(other.inMaps), - higherInDimensions(other.higherInDimensions) + higherInDimensions(other.higherInDimensions), + apparentWidth(other.apparentWidth), + apparentHeight(other.apparentHeight) { // Nothing to do. } @@ -161,9 +164,12 @@ ConvolutionType< padHTop(std::move(other.padHTop)), useBias(std::move(other.useBias)), padding(std::move(other.padding)), + paddingBackward(std::move(other.paddingBackward)), paddingType(std::move(other.paddingType)), inMaps(std::move(other.inMaps)), - higherInDimensions(std::move(other.higherInDimensions)) + higherInDimensions(std::move(other.higherInDimensions)), + apparentWidth(std::move(other.apparentWidth)), + apparentHeight(std::move(other.apparentHeight)) { // Nothing to do. } @@ -201,9 +207,12 @@ ConvolutionType< padHTop = other.padHTop; useBias = other.useBias; padding = other.padding; + paddingBackward = other.paddingBackward; paddingType = other.paddingType; inMaps = other.inMaps; higherInDimensions = other.higherInDimensions; + apparentWidth = other.apparentWidth; + apparentHeight = other.apparentHeight; } return *this; @@ -242,9 +251,12 @@ ConvolutionType< padHTop = std::move(other.padHTop); useBias = std::move(other.useBias); padding = std::move(other.padding); + paddingBackward = std::move(other.paddingBackward); paddingType = std::move(other.paddingType); inMaps = std::move(other.inMaps); higherInDimensions = std::move(other.higherInDimensions); + apparentWidth = std::move(other.apparentWidth); + apparentHeight = std::move(other.apparentHeight); } return *this; @@ -322,26 +334,27 @@ void ConvolutionType< const size_t fullOutputOffset = offset * maps; // Iterate over output maps. - for (size_t outMap = 0; outMap < maps; ++outMap) + #pragma omp parallel for + for (size_t outMap = 0; outMap < (size_t) maps; ++outMap) { + MatType& convOutput = outputTemp.slice(outMap + fullOutputOffset); // Iterate over input maps (we will apply the filter and sum). for (size_t inMap = 0; inMap < inMaps; ++inMap) { - MatType convOutput; - ForwardConvolutionRule::Convolution( inputTemp.slice(inMap + fullInputOffset), weight.slice((outMap * inMaps) + inMap), convOutput, strideWidth, - strideHeight); - - outputTemp.slice(outMap + fullOutputOffset) += convOutput; + strideHeight, + 1, + 1, + true); } // Make sure to add the bias. if (useBias) - outputTemp.slice(outMap + fullOutputOffset) += bias(outMap); + convOutput += bias(outMap); } } } @@ -374,11 +387,45 @@ void ConvolutionType< // To perform the backward pass, we need to rotate all the filters. arma::Cube rotatedFilters(weight.n_cols, weight.n_rows, weight.n_slices); - for (size_t map = 0; map < (maps * inMaps); ++map) + + // To perform the backward pass, we need to dilate all the mappedError. + arma::Cube dilatedMappedError; + if (strideHeight == 1 && strideWidth == 1) + { + MakeAlias(dilatedMappedError, mappedError.memptr(), + mappedError.n_rows, mappedError.n_cols, mappedError.n_slices); + } + else + { + dilatedMappedError.zeros(mappedError.n_rows * strideWidth - + (strideWidth - 1), mappedError.n_cols * strideHeight - + (strideHeight - 1), mappedError.n_slices); + #pragma omp parallel for collapse(3) + for (size_t i = 0; i < mappedError.n_slices; ++i) + { + for (size_t j = 0; j < mappedError.n_cols; ++j) + { + for (size_t k = 0; k < mappedError.n_rows; ++k) + { + dilatedMappedError(k * strideWidth, j * strideHeight, i) + = mappedError(k, j, i); + } + } + } + } + + #pragma omp parallel for + for (size_t map = 0; map < (size_t) (maps * inMaps); ++map) { Rotate180(weight.slice(map), rotatedFilters.slice(map)); } + MatType output(apparentWidth * apparentHeight * inMaps * higherInDimensions, + batchSize, arma::fill::zeros); + arma::Cube outputCube; + MakeAlias(outputCube, output.memptr(), apparentWidth, apparentHeight, + inMaps * higherInDimensions * batchSize); + // See Forward() for the overall iteration strategy. for (size_t offset = 0; offset < (higherInDimensions * batchSize); ++offset) { @@ -386,56 +433,43 @@ void ConvolutionType< const size_t fullOutputOffset = offset * maps; // Iterate over input maps. - for (size_t inMap = 0; inMap < inMaps; ++inMap) + #pragma omp parallel for + for (size_t inMap = 0; inMap < (size_t) inMaps; ++inMap) { // Iterate over output maps. + MatType& curG = outputCube.slice(inMap + fullInputOffset); for (size_t outMap = 0; outMap < maps; ++outMap) { - MatType output; - BackwardConvolutionRule::Convolution( - mappedError.slice(outMap + fullOutputOffset), + dilatedMappedError.slice(outMap + fullOutputOffset), rotatedFilters.slice((outMap * inMaps) + inMap), - output, - strideHeight, - strideWidth); - - // If the stride width or height is greater than 1, then we have to - // insert columns and rows into the convolution output. - if (strideWidth == 1 && strideHeight == 1) - { - if (usingPadding) - { - gTemp.slice(inMap + fullInputOffset) += output.submat( - padWLeft, - padHTop, - padWLeft + gTemp.n_rows - 1, - padHTop + gTemp.n_cols - 1); - } - else - { - gTemp.slice(inMap + fullInputOffset) += output; - } - } - else - { - // We must iterate over each element of the output and manually - // re-insert the stride. - size_t col = padWLeft; - for (size_t i = 0; i < output.n_cols; ++i) - { - size_t row = padHTop; - for (size_t j = 0; j < output.n_rows; ++j) - { - gTemp(row, col, inMap + fullInputOffset) += output(j, i); - row += strideHeight; - } - col += strideWidth; - } - } + curG, + 1, + 1, + 1, + 1, + true); } } } + MatType temp(padding.OutputDimensions()[0] * padding.OutputDimensions()[1] * + inMaps * higherInDimensions, batchSize); + arma::Cube tempCube; + MakeAlias(tempCube, temp.memptr(), padding.OutputDimensions()[0], + padding.OutputDimensions()[1], inMaps * higherInDimensions * batchSize); + paddingBackward.Forward(output, temp); + if (usingPadding) + { + gTemp = tempCube.tube( + padWLeft, + padHTop, + padWLeft + gTemp.n_rows - 1, + padHTop + gTemp.n_cols - 1); + } + else + { + gTemp = tempCube; + } } template< @@ -470,6 +504,13 @@ void ConvolutionType< const_cast(usingPadding ? inputPadded : input).memptr(), paddedRows, paddedCols, inMaps * batchSize, false, false); + MatType temp(apparentWidth * apparentHeight * inMaps * higherInDimensions, + batchSize); + arma::Cube tempCube; + MakeAlias(tempCube, temp.memptr(), apparentWidth, apparentHeight, + inMaps * higherInDimensions * batchSize); + paddingBackward.Backward(input, usingPadding ? inputPadded : input, temp); + // We will make an alias for the gradient, but note that this is only for the // convolution map weights! The bias will be handled by direct accesses into // `gradient`. @@ -483,40 +524,25 @@ void ConvolutionType< const size_t fullInputOffset = offset * inMaps; const size_t fullOutputOffset = offset * maps; - for (size_t outMap = 0; outMap < maps; ++outMap) + #pragma omp parallel for + for (size_t outMap = 0; outMap < (size_t) maps; ++outMap) { + MatType& curError = mappedError.slice(outMap + fullOutputOffset); for (size_t inMap = 0; inMap < inMaps; ++inMap) { - MatType output; GradientConvolutionRule::Convolution( - inputTemp.slice(inMap + fullInputOffset), - mappedError.slice(outMap + fullOutputOffset), - output, + tempCube.slice(inMap + fullInputOffset), + curError, + gradientTemp.slice((outMap * inMaps) + inMap), + 1, + 1, strideWidth, - strideHeight); - - // TODO: understand this conditional. Is it needed? - if (gradientTemp.n_rows < output.n_rows || - gradientTemp.n_cols < output.n_cols) - { - gradientTemp.slice((outMap * inMaps) + inMap) += output.submat(0, 0, - gradientTemp.n_rows - 1, gradientTemp.n_cols - 1); - } - else if (gradientTemp.n_rows > output.n_rows || - gradientTemp.n_cols > output.n_cols) - { - gradientTemp.slice((outMap * inMaps) + inMap).submat(0, 0, output.n_rows - 1, - output.n_cols - 1) += output; - } - else - { - gradientTemp.slice((outMap * inMaps) + inMap) += output; - } + strideHeight, + true); } if (useBias) - gradient[weight.n_elem + outMap] += arma::accu(mappedError.slice(outMap + - fullOutputOffset)); + gradient[weight.n_elem + outMap] += arma::accu(curError); } } } @@ -571,6 +597,16 @@ void ConvolutionType< this->outputDimensions[i] = this->inputDimensions[i]; } + apparentWidth = (this->outputDimensions[0] - 1) * strideWidth + kernelWidth; + apparentHeight = (this->outputDimensions[1] - 1) * strideHeight + + kernelHeight; + + paddingBackward = ann::Padding(0, padding.OutputDimensions()[0] - + apparentWidth, 0, padding.OutputDimensions()[1] - apparentHeight); + paddingBackward.InputDimensions() = std::vector({ apparentWidth, + apparentHeight, inMaps * higherInDimensions }); + paddingBackward.ComputeOutputDimensions(); + this->outputDimensions[2] = maps; } @@ -620,7 +656,7 @@ void ConvolutionType< MatType >::InitializeSamePadding() { - /* + /** * Using O = (W - F + 2P) / s + 1; */ size_t totalVerticalPadding = (strideWidth - 1) * this->inputDimensions[0] + diff --git a/src/mlpack/methods/ann/layer/grouped_convolution.hpp b/src/mlpack/methods/ann/layer/grouped_convolution.hpp index 213480cd13..b9477a3548 100644 --- a/src/mlpack/methods/ann/layer/grouped_convolution.hpp +++ b/src/mlpack/methods/ann/layer/grouped_convolution.hpp @@ -393,6 +393,9 @@ class GroupedConvolutionType : public Layer //! Locally-stored padding layer. ann::Padding padding; + //! Locally-stored padding layer for backward pass. + ann::Padding paddingBackward; + //! Type of padding. std::string paddingType; @@ -400,6 +403,12 @@ class GroupedConvolutionType : public Layer size_t inMaps; //! Locally-cached higher-order input dimensions. size_t higherInDimensions; + + //! Locally-stored apparent width. + size_t apparentWidth; + + //! Locally-stored apparent height. + size_t apparentHeight; }; // class Convolution // Standard Convolution layer. diff --git a/src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp b/src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp index 3f9ed5075f..46d50b2694 100644 --- a/src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/grouped_convolution_impl.hpp @@ -136,9 +136,12 @@ GroupedConvolutionType< padHTop(other.padHTop), useBias(other.useBias), padding(other.padding), + paddingBackward(other.paddingBackward), paddingType(other.paddingType), inMaps(other.inMaps), - higherInDimensions(other.higherInDimensions) + higherInDimensions(other.higherInDimensions), + apparentWidth(other.apparentWidth), + apparentHeight(other.apparentHeight) { // Nothing to do. } @@ -168,9 +171,12 @@ GroupedConvolutionType< padHTop(std::move(other.padHTop)), useBias(std::move(other.useBias)), padding(std::move(other.padding)), + paddingBackward(std::move(other.paddingBackward)), paddingType(std::move(other.paddingType)), inMaps(std::move(other.inMaps)), - higherInDimensions(std::move(other.higherInDimensions)) + higherInDimensions(std::move(other.higherInDimensions)), + apparentWidth(std::move(other.apparentWidth)), + apparentHeight(std::move(other.apparentHeight)) { // Nothing to do. } @@ -209,9 +215,12 @@ GroupedConvolutionType< padHTop = other.padHTop; useBias = other.useBias; padding = other.padding; + paddingBackward = other.paddingBackward; paddingType = other.paddingType; inMaps = other.inMaps; higherInDimensions = other.higherInDimensions; + apparentWidth = other.apparentWidth; + apparentHeight = other.apparentHeight; } return *this; @@ -251,9 +260,12 @@ GroupedConvolutionType< padHTop = std::move(other.padHTop); useBias = std::move(other.useBias); padding = std::move(other.padding); + paddingBackward = std::move(other.paddingBackward); paddingType = std::move(other.paddingType); inMaps = std::move(other.inMaps); higherInDimensions = std::move(other.higherInDimensions); + apparentWidth = std::move(other.apparentWidth); + apparentHeight = std::move(other.apparentHeight); } return *this; @@ -334,30 +346,32 @@ void GroupedConvolutionType< const size_t fullInputOffset = offset * inMaps; const size_t fullOutputOffset = offset * maps; + #pragma omp parallel for collapse(2) for (size_t group = 0; group < groups; group++) { // Iterate over output maps. - for (size_t outMap = group * outGroupSize; - outMap < ((group + 1) * outGroupSize); ++outMap) + for (size_t outMap = 0; outMap < outGroupSize; ++outMap) { + MatType& convOutput = outputTemp.slice(group * outGroupSize + outMap + + fullOutputOffset); // Iterate over input maps (we will apply the filter and sum). for (size_t inMap = 0; inMap < inGroupSize; ++inMap) { - MatType convOutput; - ForwardConvolutionRule::Convolution( inputTemp.slice((group * inGroupSize) + inMap + fullInputOffset), - weight.slice((outMap * inGroupSize) + inMap), + weight.slice(((group * outGroupSize + outMap) * inGroupSize) + + inMap), convOutput, strideWidth, - strideHeight); - - outputTemp.slice(outMap + fullOutputOffset) += convOutput; + strideHeight, + 1, + 1, + true); } // Make sure to add the bias. if (useBias) - outputTemp.slice(outMap + fullOutputOffset) += bias(outMap); + convOutput += bias(group * outGroupSize + outMap); } } } @@ -391,11 +405,45 @@ void GroupedConvolutionType< // To perform the backward pass, we need to rotate all the filters. arma::Cube rotatedFilters(weight.n_cols, weight.n_rows, weight.n_slices); + + #pragma omp parallel for for (size_t map = 0; map < ((maps * inMaps) / groups); ++map) { Rotate180(weight.slice(map), rotatedFilters.slice(map)); } + // To perform the backward pass, we need to dilate all the mappedError. + arma::Cube dilatedMappedError; + if (strideHeight == 1 && strideWidth == 1) + { + MakeAlias(dilatedMappedError, mappedError.memptr(), + mappedError.n_rows, mappedError.n_cols, mappedError.n_slices); + } + else + { + dilatedMappedError.zeros(mappedError.n_rows * strideWidth - + (strideWidth - 1), mappedError.n_cols * strideHeight - + (strideHeight - 1), mappedError.n_slices); + #pragma omp parallel for collapse(3) + for (size_t i = 0; i < mappedError.n_slices; ++i) + { + for (size_t j = 0; j < mappedError.n_cols; ++j) + { + for (size_t k = 0; k < mappedError.n_rows; ++k) + { + dilatedMappedError(k * strideWidth, j * strideHeight, i) + = mappedError(k, j, i); + } + } + } + } + + MatType output(apparentWidth * apparentHeight * inMaps * higherInDimensions, + batchSize, arma::fill::zeros); + arma::Cube outputCube; + MakeAlias(outputCube, output.memptr(), apparentWidth, apparentHeight, + inMaps * higherInDimensions * batchSize); + size_t inGroupSize = inMaps / groups; size_t outGroupSize = maps / groups; @@ -405,64 +453,50 @@ void GroupedConvolutionType< const size_t fullInputOffset = offset * inMaps; const size_t fullOutputOffset = offset * maps; + #pragma omp parallel for collapse(2) for (size_t group = 0; group < groups; group++) { - // Iterate over input maps. + // Iterate over input maps. for (size_t inMap = 0; inMap < inGroupSize; ++inMap) { // Iterate over output maps. - for (size_t outMap = group * outGroupSize; - outMap < ((group + 1) * outGroupSize); ++outMap) + MatType& curG = outputCube.slice((group * inGroupSize) + inMap + + fullInputOffset); + for (size_t outMap = group * outGroupSize; outMap < (group + 1) * + outGroupSize; ++outMap) { - MatType output; - BackwardConvolutionRule::Convolution( - mappedError.slice(outMap + fullOutputOffset), + dilatedMappedError.slice(outMap + fullOutputOffset), rotatedFilters.slice((outMap * inGroupSize) + inMap), - output, - strideHeight, - strideWidth); - - // If the stride width or height is greater than 1, then we have to - // insert columns and rows into the convolution output. - if (strideWidth == 1 && strideHeight == 1) - { - if (usingPadding) - { - gTemp.slice((group * inGroupSize) + inMap + fullInputOffset) += - output.submat( - padWLeft, - padHTop, - padWLeft + gTemp.n_rows - 1, - padHTop + gTemp.n_cols - 1); - } - else - { - gTemp.slice((group * inGroupSize) + inMap + fullInputOffset) += - output; - } - } - else - { - // We must iterate over each element of the output and manually - // re-insert the stride. - size_t col = padWLeft; - for (size_t i = 0; i < output.n_cols; ++i) - { - size_t row = padHTop; - for (size_t j = 0; j < output.n_rows; ++j) - { - gTemp(row, col, (group * inGroupSize) + inMap + - fullInputOffset) += output(j, i); - row += strideHeight; - } - col += strideWidth; - } - } + curG, + 1, + 1, + 1, + 1, + true); } } } } + + MatType temp(padding.OutputDimensions()[0] * padding.OutputDimensions()[1] * + inMaps * higherInDimensions, batchSize); + arma::Cube tempCube; + MakeAlias(tempCube, temp.memptr(), padding.OutputDimensions()[0], + padding.OutputDimensions()[1], inMaps * higherInDimensions * batchSize); + paddingBackward.Forward(output, temp); + if (usingPadding) + { + gTemp = tempCube.tube( + padWLeft, + padHTop, + padWLeft + gTemp.n_rows - 1, + padHTop + gTemp.n_cols - 1); + } + else + { + gTemp = tempCube; + } } template< @@ -497,6 +531,13 @@ void GroupedConvolutionType< const_cast(usingPadding ? inputPadded : input).memptr(), paddedRows, paddedCols, inMaps * batchSize, false, false); + MatType temp(apparentWidth * apparentHeight * inMaps * higherInDimensions, + batchSize); + arma::Cube tempCube; + MakeAlias(tempCube, temp.memptr(), apparentWidth, apparentHeight, + inMaps * higherInDimensions * batchSize); + paddingBackward.Backward(input, usingPadding ? inputPadded : input, temp); + // We will make an alias for the gradient, but note that this is only for the // convolution map weights! The bias will be handled by direct accesses into // `gradient`. @@ -513,49 +554,33 @@ void GroupedConvolutionType< const size_t fullInputOffset = offset * inMaps; const size_t fullOutputOffset = offset * maps; + #pragma omp parallel for collapse(2) for (size_t group = 0; group < groups; group++) { // Iterate over output maps. - for (size_t outMap = group * outGroupSize; - outMap < ((group + 1) * outGroupSize); ++outMap) + for (size_t outMap = 0; outMap < outGroupSize; ++outMap) { // Iterate over input maps (we will apply the filter and sum). + MatType& curError = mappedError.slice(group * outGroupSize + outMap + + fullOutputOffset); for (size_t inMap = 0; inMap < inGroupSize; ++inMap) { MatType output; GradientConvolutionRule::Convolution( - inputTemp.slice((group * inGroupSize) + inMap + fullInputOffset), - mappedError.slice(outMap + fullOutputOffset), - output, + tempCube.slice((group * inGroupSize) + inMap + fullInputOffset), + curError, + gradientTemp.slice(((group * outGroupSize + outMap) * + inGroupSize) + inMap), + 1, + 1, strideWidth, - strideHeight); - - // TODO: understand this conditional. Is it needed? - if (gradientTemp.n_rows < output.n_rows || - gradientTemp.n_cols < output.n_cols) - { - gradientTemp.slice((outMap * inGroupSize) + inMap) += - output.submat( - 0, - 0, - gradientTemp.n_rows - 1, - gradientTemp.n_cols - 1); - } - else if (gradientTemp.n_rows > output.n_rows || - gradientTemp.n_cols > output.n_cols) - { - gradientTemp.slice((outMap * inGroupSize) + inMap).submat(0, 0, - output.n_rows - 1, output.n_cols - 1) += output; - } - else - { - gradientTemp.slice((outMap * inGroupSize) + inMap) += output; - } + strideHeight, + true); } if (useBias) - gradient[weight.n_elem + outMap] += arma::accu(mappedError.slice( - outMap + fullOutputOffset)); + gradient[weight.n_elem + group * outGroupSize + outMap] += + arma::accu(curError); } } } @@ -610,8 +635,8 @@ void GroupedConvolutionType< if ((inMaps % groups != 0) || (maps % groups != 0)) { Log::Fatal << "GroupedConvolution::ComputeOutputDimensions(): both input " - << "maps (" << inMaps << ") and output maps (" << maps << ") should be " - << "divisible by groups (" << groups << ")!" << std::endl; + << "maps (" << inMaps << ") and output maps (" << maps << ") should be" + << " divisible by groups (" << groups << ")!" << std::endl; } // Compute and cache the total number of input maps. @@ -622,6 +647,16 @@ void GroupedConvolutionType< this->outputDimensions[i] = this->inputDimensions[i]; } + apparentWidth = (this->outputDimensions[0] - 1) * strideWidth + kernelWidth; + apparentHeight = (this->outputDimensions[1] - 1) * strideHeight + + kernelHeight; + + paddingBackward = ann::Padding(0, padding.OutputDimensions()[0] - + apparentWidth, 0, padding.OutputDimensions()[1] - apparentHeight); + paddingBackward.InputDimensions() = std::vector({ apparentWidth, + apparentHeight, inMaps * higherInDimensions }); + paddingBackward.ComputeOutputDimensions(); + this->outputDimensions[2] = maps; } diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index c50a667580..a0984f2eee 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -57,7 +57,11 @@ // Loss function modules. #include -// Include definitions for polymorphic serialization. +// Include definitions for polymorphic serialization. Note that this can cause +// significant compilation overhead, so we only do it if +// MLPACK_ENABLE_ANN_SERIALIZATION is enabled. +#ifdef MLPACK_ENABLE_ANN_SERIALIZATION #include +#endif #endif diff --git a/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp b/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp index 7a6e8dce3a..dd8a983a8a 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp @@ -74,19 +74,18 @@ LeakyReLUType::operator=(LeakyReLUType&& other) template void LeakyReLUType::Forward(const MatType& input, MatType& output) { - output = arma::max(input, alpha * input); + #pragma omp for + for (size_t i = 0; i < (size_t) input.n_elem; ++i) + output(i) = std::max(input(i), alpha * input(i)); } template void LeakyReLUType::Backward( const MatType& input, const MatType& gy, MatType& g) { - MatType derivative; - derivative.set_size(arma::size(input)); - for (size_t i = 0; i < input.n_elem; ++i) - derivative(i) = (input(i) >= 0) ? 1 : alpha; - - g = gy % derivative; + #pragma omp for + for (size_t i = 0; i < (size_t) input.n_elem; ++i) + g(i) = gy(i) * ((input(i) >= 0) ? 1 : alpha); } template diff --git a/src/mlpack/methods/ann/layer/linear_impl.hpp b/src/mlpack/methods/ann/layer/linear_impl.hpp index fe7841e791..c0523f602e 100644 --- a/src/mlpack/methods/ann/layer/linear_impl.hpp +++ b/src/mlpack/methods/ann/layer/linear_impl.hpp @@ -107,7 +107,10 @@ void LinearType::Forward( const MatType& input, MatType& output) { output = weight * input; - output.each_col() += bias; + + #pragma omp for + for (size_t c = 0; c < (size_t) output.n_cols; ++c) + output.col(c) += bias; } template diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index e9f009c3c4..b809278cfb 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -165,7 +165,8 @@ class MaxPoolingType : public Layer arma::Cube& poolingIndices) { // Iterate over all slices individually. - for (size_t s = 0; s < input.n_slices; ++s) + #pragma omp parallel for + for (size_t s = 0; s < (size_t) input.n_slices; ++s) { for (size_t j = 0, colidx = 0; j < output.n_cols; ++j, colidx += strideHeight) @@ -204,8 +205,7 @@ class MaxPoolingType : public Layer const size_t poolingCol = poolIndex / (kernelWidth); const size_t poolingRow = poolIndex % (kernelWidth); const size_t unmappedPoolingIndex = (rowidx + poolingRow) + - input.n_rows * (colidx + poolingCol) + - input.n_rows * input.n_cols * s; + input.n_rows * (colidx + poolingCol); poolingIndices(i, j, s) = unmappedPoolingIndex; output(i, j, s) = std::get<1>(poolResult); @@ -226,7 +226,8 @@ class MaxPoolingType : public Layer arma::Cube& output) { // Iterate over all slices individually. - for (size_t s = 0; s < input.n_slices; ++s) + #pragma omp parallel for + for (size_t s = 0; s < (size_t) input.n_slices; ++s) { for (size_t j = 0, colidx = 0; j < output.n_cols; ++j, colidx += strideHeight) @@ -271,12 +272,10 @@ class MaxPoolingType : public Layer * @param poolingIndices The pooled indices (from `PoolingOperation()`). */ void UnpoolingOperation( - const arma::Cube& error, - arma::Cube& output, - const arma::Cube& poolingIndices) + const MatType& error, + MatType& output, + const arma::Mat& poolingIndices) { - output.zeros(); - for (size_t i = 0; i < poolingIndices.n_elem; ++i) { output(poolingIndices(i)) += error(i); diff --git a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp index 1128f3a4df..1326e829b2 100644 --- a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp @@ -151,9 +151,15 @@ void MaxPoolingType::Backward( this->inputDimensions[0], this->inputDimensions[1], channels * input.n_cols, false, true); + gTemp.zeros(); + // There's no version of UnpoolingOperation without pooling indices, because // if we call `Backward()`, we know for sure we are training. - UnpoolingOperation(mappedError, gTemp, poolingIndices); + #pragma omp parallel for + for (size_t s = 0; s < (size_t) mappedError.n_slices; s++) + { + UnpoolingOperation(mappedError.slice(s), gTemp.slice(s), poolingIndices.slice(s)); + } } template diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index e56c1b9d67..39524c032c 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -146,7 +146,8 @@ void MeanPoolingType::Backward( // Initialize the gradient with zero. gTemp.zeros(); - for (size_t s = 0; s < mappedError.n_slices; s++) + #pragma omp parallel for + for (size_t s = 0; s < (size_t) mappedError.n_slices; s++) { // Computing gradient of each slice. Unpooling(mappedError.slice(s), gTemp.slice(s)); @@ -204,7 +205,8 @@ void MeanPoolingType::PoolingOperation( arma::Cube& output) { // Iterate over all slices individually. - for (size_t s = 0; s < input.n_slices; ++s) + #pragma omp parallel for + for (size_t s = 0; s < (size_t) input.n_slices; ++s) { for (size_t j = 0, colidx = 0; j < output.n_cols; ++j, colidx += strideHeight) diff --git a/src/mlpack/methods/ann/layer/padding_impl.hpp b/src/mlpack/methods/ann/layer/padding_impl.hpp index 69aa6dde60..ef48f43992 100644 --- a/src/mlpack/methods/ann/layer/padding_impl.hpp +++ b/src/mlpack/methods/ann/layer/padding_impl.hpp @@ -107,43 +107,43 @@ void PaddingType::Forward(const MatType& input, MatType& output) output.n_cols, false, true); // Set the padding parts to 0. - if (padWLeft > 0) + if (padHTop > 0) { reshapedOutput.tube(0, 0, reshapedOutput.n_rows - 1, - padWLeft - 1).zeros(); + padHTop - 1).zeros(); } - if (padHTop > 0) + if (padWLeft > 0) { reshapedOutput.tube(0, - padWLeft, - padHTop - 1, - padWLeft + this->inputDimensions[1] - 1).zeros(); - } - - if (padWRight > 0) - { - reshapedOutput.tube(0, - padWLeft + this->inputDimensions[1], - reshapedOutput.n_rows - 1, - reshapedOutput.n_cols - 1).zeros(); + padHTop, + padWLeft - 1, + padHTop + this->inputDimensions[1] - 1).zeros(); } if (padHBottom > 0) { - reshapedOutput.tube(padHTop + this->inputDimensions[0], - padWLeft, + reshapedOutput.tube(0, + padHTop + this->inputDimensions[1], reshapedOutput.n_rows - 1, - padWLeft + this->inputDimensions[1] - 1).zeros(); + reshapedOutput.n_cols - 1).zeros(); + } + + if (padWRight > 0) + { + reshapedOutput.tube(padWLeft + this->inputDimensions[0], + padHTop, + reshapedOutput.n_rows - 1, + padHTop + this->inputDimensions[1] - 1).zeros(); } // Copy the input matrix. - reshapedOutput.tube(padHTop, - padWLeft, - padHTop + this->inputDimensions[0] - 1, - padWLeft + this->inputDimensions[1] - 1) = reshapedInput; + reshapedOutput.tube(padWLeft, + padHTop, + padWLeft + this->inputDimensions[0] - 1, + padHTop + this->inputDimensions[1] - 1) = reshapedInput; } template @@ -161,10 +161,10 @@ void PaddingType::Backward( this->inputDimensions[0], this->inputDimensions[1], totalInMaps * g.n_cols, false, true); - reshapedG = reshapedGy.tube(padHTop, - padWLeft, - padHTop + this->inputDimensions[0] - 1, - padWLeft + this->inputDimensions[1] - 1); + reshapedG = reshapedGy.tube(padWLeft, + padHTop, + padWLeft + this->inputDimensions[0] - 1, + padHTop + this->inputDimensions[1] - 1); } template @@ -172,8 +172,8 @@ void PaddingType::ComputeOutputDimensions() { this->outputDimensions = this->inputDimensions; - this->outputDimensions[0] += padHTop + padHBottom; - this->outputDimensions[1] += padWLeft + padWRight; + this->outputDimensions[0] += padWLeft + padWRight; + this->outputDimensions[1] += padHTop + padHBottom; // Higher dimensions remain unchanged. But, we will cache the product of // these higher dimensions. diff --git a/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp b/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp index 4f0897a9ad..6effcfcd19 100644 --- a/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp +++ b/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp @@ -118,9 +118,9 @@ BINDING_SEE_ALSO("Approximate furthest neighbor in high dimensions (pdf)", "https://pdfs.semanticscholar.org/a4b5/7b9cbf37201fb1d9a56c0f4eefad0466" "9c20.pdf"); BINDING_SEE_ALSO("mlpack::neighbor::QDAFN class documentation", - "@doxygen/classmlpack_1_1neighbor_1_1QDAFN.html"); + "@src/mlpack/methods/approx_kfn/qdafn.hpp."); BINDING_SEE_ALSO("mlpack::neighbor::DrusillaSelect class documentation", - "@doxygen/classmlpack_1_1neighbor_1_1DrusillaSelect.html"); + "@src/mlpack/methods/approx_kfn/drusilla_select.hpp"); PARAM_MATRIX_IN("reference", "Matrix containing the reference dataset.", "r"); PARAM_MATRIX_IN("query", "Matrix containing query points.", "q"); diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_main.cpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_main.cpp index 7cb0657ce4..b9f84f0515 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_main.cpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_main.cpp @@ -103,8 +103,8 @@ BINDING_SEE_ALSO("Bayesian Linear Regression, Section 3.3", "MLA Bishop, Christopher M. Pattern Recognition and Machine " "Learning. New York :Springer, 2006, section 3.3."); BINDING_SEE_ALSO("mlpack::regression::BayesianLinearRegression C++ class " - "documentation", - "@doxygen/classmlpack_1_1regression_1_1BayesianLinearRegression.html"); + "documentation", "@src/mlpack/methods/bayesian_linear_regression/" + "bayesian_linear_regression.hpp"); PARAM_MATRIX_IN("input", "Matrix of covariates (X).", "i"); diff --git a/src/mlpack/methods/bias_svd/bias_svd_function.hpp b/src/mlpack/methods/bias_svd/bias_svd_function.hpp index 43978bed4b..7eb0496c7c 100644 --- a/src/mlpack/methods/bias_svd/bias_svd_function.hpp +++ b/src/mlpack/methods/bias_svd/bias_svd_function.hpp @@ -139,10 +139,6 @@ class BiasSVDFunction } // namespace svd } // namespace mlpack -/** - * @cond NO_DOXYGEN - */ - namespace ens { /** @@ -165,10 +161,6 @@ namespace ens { } // namespace ens -/** - * @endcond - */ - #include "bias_svd_function_impl.hpp" #endif diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp index 35f4a59d0f..e89309ba9f 100644 --- a/src/mlpack/methods/cf/cf_main.cpp +++ b/src/mlpack/methods/cf/cf_main.cpp @@ -122,9 +122,9 @@ BINDING_EXAMPLE( // See also... BINDING_SEE_ALSO("Collaborative filtering tutorial", - "@doxygen/cftutorial.html"); + "@doc/tutorials/cf.md"); BINDING_SEE_ALSO("Alternating Matrix Factorization tutorial", - "@doxygen/amftutorial.html"); + "@doc/tutorials/amf.md"); BINDING_SEE_ALSO("Collaborative Filtering on Wikipedia", "https://en.wikipedia.org/wiki/Collaborative_filtering"); BINDING_SEE_ALSO("Matrix factorization on Wikipedia", @@ -134,7 +134,7 @@ BINDING_SEE_ALSO("Matrix factorization techniques for recommender systems" " (pdf)", "http://citeseerx.ist.psu.edu/viewdoc/download?doi=" "10.1.1.441.3234&rep=rep1&type=pdf"); BINDING_SEE_ALSO("mlpack::cf::CFType class documentation", - "@doxygen/classmlpack_1_1cf_1_1CFType.html"); + "@src/mlpack/methods/cf/cf.hpp"); // Parameters for training a model. PARAM_MATRIX_IN("training", "Input dataset to perform CF on.", "t"); diff --git a/src/mlpack/methods/cf/decomposition_policies/regularized_svd_method.hpp b/src/mlpack/methods/cf/decomposition_policies/regularized_svd_method.hpp index f753f8e6b6..7ca95ad336 100644 --- a/src/mlpack/methods/cf/decomposition_policies/regularized_svd_method.hpp +++ b/src/mlpack/methods/cf/decomposition_policies/regularized_svd_method.hpp @@ -45,7 +45,7 @@ class RegSVDPolicy * Use regularized SVD method to perform collaborative filtering. * * @param maxIterations Number of iterations for the power method - * (Default: 2). + * (Default: 10). */ RegSVDPolicy(const size_t maxIterations = 10) : maxIterations(maxIterations) diff --git a/src/mlpack/methods/dbscan/dbscan_main.cpp b/src/mlpack/methods/dbscan/dbscan_main.cpp index beb650fbb6..0984284d3c 100644 --- a/src/mlpack/methods/dbscan/dbscan_main.cpp +++ b/src/mlpack/methods/dbscan/dbscan_main.cpp @@ -77,7 +77,7 @@ BINDING_SEE_ALSO("A density-based algorithm for discovering clusters in large " "spatial databases with noise (pdf)", "http://www.aaai.org/Papers/KDD/1996/KDD96-037.pdf"); BINDING_SEE_ALSO("mlpack::dbscan::DBSCAN class documentation", - "@doxygen/classmlpack_1_1dbscan_1_1DBSCAN.html"); + "@src/mlpack/methods/dbscan/dbscan.hpp"); PARAM_MATRIX_IN_REQ("input", "Input dataset to cluster.", "i"); PARAM_UROW_OUT("assignments", "Output matrix for assignments of each " diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index 2fd35dd67b..e4bdf52b7f 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -98,7 +98,7 @@ BINDING_SEE_ALSO("Decision trees on Wikipedia", BINDING_SEE_ALSO("Induction of Decision Trees (pdf)", "https://link.springer.com/content/pdf/10.1007/BF00116251.pdf"); BINDING_SEE_ALSO("mlpack::tree::DecisionTree class documentation", - "@doxygen/classmlpack_1_1tree_1_1DecisionTree.html"); + "@src/mlpack/methods/decision_tree/decision_tree.hpp"); // Datasets. PARAM_MATRIX_AND_INFO_IN("training", "Training dataset (may be categorical).", diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index b63925832b..9cd0be968e 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -66,13 +66,13 @@ BINDING_LONG_DESC( // See also... BINDING_SEE_ALSO("Density estimation tree (DET) tutorial", - "@doxygen/dettutorial.html"); + "@doc/tutorials/det.md"); BINDING_SEE_ALSO("Density estimation on Wikipedia", "https://en.wikipedia.org/wiki/Density_estimation"); BINDING_SEE_ALSO("Density estimation trees (pdf)", "http://www.mlpack.org/papers/det.pdf"); BINDING_SEE_ALSO("mlpack::tree::DTree class documentation", - "@doxygen/classmlpack_1_1det_1_1DTree.html"); + "@src/mlpack/methods/det/dtree.hpp"); // Input data files. PARAM_MATRIX_IN("training", "The data set on which to build a density " diff --git a/src/mlpack/methods/emst/emst_main.cpp b/src/mlpack/methods/emst/emst_main.cpp index 61992817f8..be513efe53 100644 --- a/src/mlpack/methods/emst/emst_main.cpp +++ b/src/mlpack/methods/emst/emst_main.cpp @@ -73,13 +73,13 @@ BINDING_EXAMPLE( "the third column corresponds to the distance between the two points."); // See also... -BINDING_SEE_ALSO("EMST Tutorial", "@doxygen/emst_tutorial.html"); +BINDING_SEE_ALSO("EMST Tutorial", "@doc/tutorials/emst.md"); BINDING_SEE_ALSO("Minimum spanning tree on Wikipedia", "https://en.wikipedia.org/wiki/Minimum_spanning_tree"); BINDING_SEE_ALSO("Fast Euclidean Minimum Spanning Tree: Algorithm, Analysis," " and Applications (pdf)", "http://www.mlpack.org/papers/emst.pdf"); BINDING_SEE_ALSO("mlpack::emst::DualTreeBoruvka class documentation", - "@doxygen/classmlpack_1_1emst_1_1DualTreeBoruvka.html"); + "@src/mlpack/methods/emst/dtb.hpp"); PARAM_MATRIX_IN_REQ("input", "Input data matrix.", "i"); PARAM_MATRIX_OUT("output", "Output data. Stored as an edge list.", "o"); diff --git a/src/mlpack/methods/fastmks/fastmks_main.cpp b/src/mlpack/methods/fastmks/fastmks_main.cpp index 862123b789..8ff4e1b877 100644 --- a/src/mlpack/methods/fastmks/fastmks_main.cpp +++ b/src/mlpack/methods/fastmks/fastmks_main.cpp @@ -71,12 +71,12 @@ BINDING_EXAMPLE( // See also... BINDING_SEE_ALSO("Fast max-kernel search tutorial (fastmks)", - "@doxygen/fmkstutorial.html"); + "@doc/tutorials/fastmks.md"); BINDING_SEE_ALSO("k-nearest-neighbor search", "#knn"); BINDING_SEE_ALSO("Dual-tree Fast Exact Max-Kernel Search (pdf)", "http://mlpack.org/papers/fmks.pdf"); BINDING_SEE_ALSO("mlpack::fastmks::FastMKS class documentation", - "@doxygen/classmlpack_1_1fastmks_1_1FastMKS.html"); + "@src/mlpack/methods/fastmks/fastmks.hpp"); // Model-building parameters. PARAM_MATRIX_IN("reference", "The reference dataset.", "r"); diff --git a/src/mlpack/methods/gmm/gmm_generate_main.cpp b/src/mlpack/methods/gmm/gmm_generate_main.cpp index 0df2117cc1..e9c426708b 100644 --- a/src/mlpack/methods/gmm/gmm_generate_main.cpp +++ b/src/mlpack/methods/gmm/gmm_generate_main.cpp @@ -55,7 +55,7 @@ BINDING_SEE_ALSO("@gmm_probability", "#gmm_probability"); BINDING_SEE_ALSO("Gaussian Mixture Models on Wikipedia", "https://en.wikipedia.org/wiki/Mixture_model#Gaussian_mixture_model"); BINDING_SEE_ALSO("mlpack::gmm::GMM class documentation", - "@doxygen/classmlpack_1_1gmm_1_1GMM.html"); + "@src/mlpack/methods/gmm/gmm.hpp"); PARAM_MODEL_IN_REQ(GMM, "input_model", "Input GMM model to generate samples " "from.", "m"); diff --git a/src/mlpack/methods/gmm/gmm_probability_main.cpp b/src/mlpack/methods/gmm/gmm_probability_main.cpp index 77eaf93c4d..e58eeb02c1 100644 --- a/src/mlpack/methods/gmm/gmm_probability_main.cpp +++ b/src/mlpack/methods/gmm/gmm_probability_main.cpp @@ -56,7 +56,7 @@ BINDING_SEE_ALSO("@gmm_generate", "#gmm_generate"); BINDING_SEE_ALSO("Gaussian Mixture Models on Wikipedia", "https://en.wikipedia.org/wiki/Mixture_model#Gaussian_mixture_model"); BINDING_SEE_ALSO("mlpack::gmm::GMM class documentation", - "@doxygen/classmlpack_1_1gmm_1_1GMM.html"); + "@src/mlpack/methods/gmm/gmm.hpp"); PARAM_MODEL_IN_REQ(GMM, "input_model", "Input GMM to use as model.", "m"); PARAM_MATRIX_IN_REQ("input", "Input matrix to calculate probabilities of.", diff --git a/src/mlpack/methods/gmm/gmm_train_main.cpp b/src/mlpack/methods/gmm/gmm_train_main.cpp index 23b6b088f4..a3a2382545 100644 --- a/src/mlpack/methods/gmm/gmm_train_main.cpp +++ b/src/mlpack/methods/gmm/gmm_train_main.cpp @@ -110,7 +110,7 @@ BINDING_SEE_ALSO("@gmm_probability", "#gmm_probability"); BINDING_SEE_ALSO("Gaussian Mixture Models on Wikipedia", "https://en.wikipedia.org/wiki/Mixture_model#Gaussian_mixture_model"); BINDING_SEE_ALSO("mlpack::gmm::GMM class documentation", - "@doxygen/classmlpack_1_1gmm_1_1GMM.html"); + "@src/mlpack/methods/gmm/gmm.hpp"); // Parameters for training. PARAM_MATRIX_IN_REQ("input", "The training data on which the model will be " diff --git a/src/mlpack/methods/hmm/hmm_generate_main.cpp b/src/mlpack/methods/hmm/hmm_generate_main.cpp index 01e2f92435..f6d04f331d 100644 --- a/src/mlpack/methods/hmm/hmm_generate_main.cpp +++ b/src/mlpack/methods/hmm/hmm_generate_main.cpp @@ -71,7 +71,7 @@ BINDING_SEE_ALSO("@hmm_viterbi", "#hmm_viterbi"); BINDING_SEE_ALSO("Hidden Mixture Models on Wikipedia", "https://en.wikipedia.org/wiki/Hidden_Markov_model"); BINDING_SEE_ALSO("mlpack::hmm::HMM class documentation", - "@doxygen/classmlpack_1_1hmm_1_1HMM.html"); + "@src/mlpack/methods/hmm/hmm.hpp"); PARAM_MODEL_IN_REQ(HMMModel, "model", "Trained HMM to generate sequences with.", "m"); diff --git a/src/mlpack/methods/hmm/hmm_loglik_main.cpp b/src/mlpack/methods/hmm/hmm_loglik_main.cpp index 147063e867..da55d942b1 100644 --- a/src/mlpack/methods/hmm/hmm_loglik_main.cpp +++ b/src/mlpack/methods/hmm/hmm_loglik_main.cpp @@ -62,7 +62,7 @@ BINDING_SEE_ALSO("@hmm_viterbi", "#hmm_viterbi"); BINDING_SEE_ALSO("Hidden Mixture Models on Wikipedia", "https://en.wikipedia.org/wiki/Hidden_Markov_model"); BINDING_SEE_ALSO("mlpack::hmm::HMM class documentation", - "@doxygen/classmlpack_1_1hmm_1_1HMM.html"); + "@src/mlpack/methods/hmm/hmm.hpp"); PARAM_MATRIX_IN_REQ("input", "File containing observations,", "i"); PARAM_MODEL_IN_REQ(HMMModel, "input_model", "File containing HMM.", "m"); diff --git a/src/mlpack/methods/hmm/hmm_train_main.cpp b/src/mlpack/methods/hmm/hmm_train_main.cpp index ff358d3be2..d9942ccb4e 100644 --- a/src/mlpack/methods/hmm/hmm_train_main.cpp +++ b/src/mlpack/methods/hmm/hmm_train_main.cpp @@ -73,7 +73,7 @@ BINDING_SEE_ALSO("@hmm_viterbi", "#hmm_viterbi"); BINDING_SEE_ALSO("Hidden Mixture Models on Wikipedia", "https://en.wikipedia.org/wiki/Hidden_Markov_model"); BINDING_SEE_ALSO("mlpack::hmm::HMM class documentation", - "@doxygen/classmlpack_1_1hmm_1_1HMM.html"); + "@src/mlpack/methods/hmm/hmm.hpp"); PARAM_STRING_IN_REQ("input_file", "File containing input observations.", "i"); PARAM_STRING_IN("type", "Type of HMM: discrete | gaussian | diag_gmm | gmm.", diff --git a/src/mlpack/methods/hmm/hmm_viterbi_main.cpp b/src/mlpack/methods/hmm/hmm_viterbi_main.cpp index 32900d3c47..cc0a5b0cf0 100644 --- a/src/mlpack/methods/hmm/hmm_viterbi_main.cpp +++ b/src/mlpack/methods/hmm/hmm_viterbi_main.cpp @@ -66,7 +66,7 @@ BINDING_SEE_ALSO("@hmm_loglik", "#hmm_loglik"); BINDING_SEE_ALSO("Hidden Mixture Models on Wikipedia", "https://en.wikipedia.org/wiki/Hidden_Markov_model"); BINDING_SEE_ALSO("mlpack::hmm::HMM class documentation", - "@doxygen/classmlpack_1_1hmm_1_1HMM.html"); + "@src/mlpack/methods/hmm/hmm.hpp"); PARAM_MATRIX_IN_REQ("input", "Matrix containing observations,", "i"); PARAM_MODEL_IN_REQ(HMMModel, "input_model", "Trained HMM to use.", "m"); diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp index 50a229eb7f..e1cbc6bd9c 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp @@ -93,7 +93,7 @@ BINDING_SEE_ALSO("@random_forest", "#random_forest"); BINDING_SEE_ALSO("Mining High-Speed Data Streams (pdf)", "http://dm.cs.washington.edu/papers/vfdt-kdd00.pdf"); BINDING_SEE_ALSO("mlpack::tree::HoeffdingTree class documentation", - "@doxygen/classmlpack_1_1tree_1_1HoeffdingTree.html"); + "@src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp"); PARAM_MATRIX_AND_INFO_IN("training", "Training dataset (may be categorical).", "t"); diff --git a/src/mlpack/methods/kde/kde_main.cpp b/src/mlpack/methods/kde/kde_main.cpp index 55b946b0df..19bcb452a4 100644 --- a/src/mlpack/methods/kde/kde_main.cpp +++ b/src/mlpack/methods/kde/kde_main.cpp @@ -129,7 +129,7 @@ BINDING_SEE_ALSO("Fast High-dimensional Kernel Summations Using the Monte Carlo" "dimensional-kernel-summations-using-the-monte-carlo-multipole-method." "pdf"); BINDING_SEE_ALSO("mlpack::kde::KDE C++ class documentation", - "@doxygen/classmlpack_1_1kde_1_1KDE.html"); + "@src/mlpack/methods/kde/kde.hpp"); // Required options. PARAM_MATRIX_IN("reference", "Input reference dataset use for KDE.", "r"); diff --git a/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp b/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp index 07f2669b6c..4eccef7860 100644 --- a/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp +++ b/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp @@ -101,7 +101,7 @@ BINDING_SEE_ALSO("Kernel principal component analysis on Wikipedia", BINDING_SEE_ALSO("Kernel Principal Component Analysis (pdf)", "http://pca.narod.ru/scholkopf_kernel.pdf"); BINDING_SEE_ALSO("mlpack::kpca::KernelPCA class documentation", - "@doxygen/classmlpack_1_1kpca_1_1KernelPCA.html"); + "@src/mlpack/methods/kernel_pca/kernel_pca.hpp"); PARAM_MATRIX_IN_REQ("input", "Input dataset to perform KPCA on.", "i"); PARAM_MATRIX_OUT("output", "Matrix to save modified dataset to.", "o"); diff --git a/src/mlpack/methods/kmeans/kmeans_main.cpp b/src/mlpack/methods/kmeans/kmeans_main.cpp index 99c43e0168..55a533edf5 100644 --- a/src/mlpack/methods/kmeans/kmeans_main.cpp +++ b/src/mlpack/methods/kmeans/kmeans_main.cpp @@ -108,7 +108,7 @@ BINDING_EXAMPLE( "clusters", 10, "max_iterations", 500, "centroid", "final")); // See also... -BINDING_SEE_ALSO("K-Means tutorial", "@doxygen/kmtutorial.html"); +BINDING_SEE_ALSO("K-Means tutorial", "@doc/tutorials/kmeans.md"); BINDING_SEE_ALSO("@dbscan", "#dbscan"); BINDING_SEE_ALSO("k-means++", "https://en.wikipedia.org/wiki/K-means%2B%2B"); BINDING_SEE_ALSO("Using the triangle inequality to accelerate k-means (pdf)", @@ -122,7 +122,7 @@ BINDING_SEE_ALSO("Accelerating exact k-means algorithms with geometric" BINDING_SEE_ALSO("A dual-tree algorithm for fast k-means clustering with large " "k (pdf)", "http://www.ratml.org/pub/pdf/2017dual.pdf"); BINDING_SEE_ALSO("mlpack::kmeans::KMeans class documentation", - "@doxygen/classmlpack_1_1kmeans_1_1KMeans.html"); + "@src/mlpack/methods/kmeans/kmeans.hpp"); // Required options. PARAM_MATRIX_IN_REQ("input", "Input dataset to perform clustering on.", "i"); diff --git a/src/mlpack/methods/lars/lars_main.cpp b/src/mlpack/methods/lars/lars_main.cpp index 6fd22cb173..46fff31d91 100644 --- a/src/mlpack/methods/lars/lars_main.cpp +++ b/src/mlpack/methods/lars/lars_main.cpp @@ -103,7 +103,7 @@ BINDING_SEE_ALSO("@linear_regression", "#linear_regression"); BINDING_SEE_ALSO("Least angle regression (pdf)", "http://mlpack.org/papers/lars.pdf"); BINDING_SEE_ALSO("mlpack::regression::LARS C++ class documentation", - "@doxygen/classmlpack_1_1regression_1_1LARS.html"); + "@src/mlpack/methods/lars/lars.hpp"); PARAM_TMATRIX_IN("input", "Matrix of covariates (X).", "i"); PARAM_MATRIX_IN("responses", "Matrix of responses/observations (y).", "r"); diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp index 7399e5770c..92a7a0e5cd 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp @@ -83,13 +83,13 @@ BINDING_EXAMPLE( // See also... BINDING_SEE_ALSO("Linear/ridge regression tutorial", - "@doxygen/lrtutorial.html"); + "@doc/tutorials/linear_regression.md"); BINDING_SEE_ALSO("@lars", "#lars"); BINDING_SEE_ALSO("Linear regression on Wikipedia", "https://en.wikipedia.org/wiki/Linear_regression"); BINDING_SEE_ALSO("mlpack::regression::LinearRegression C++ class " "documentation", - "@doxygen/classmlpack_1_1regression_1_1LinearRegression.html"); + "@src/mlpack/methods/linear_regression/linear_regression.hpp"); PARAM_MATRIX_IN("training", "Matrix containing training set X (regressors).", "t"); diff --git a/src/mlpack/methods/linear_regression/linear_regression_train_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_train_main.cpp index 82ca85b3f4..e88fd80509 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_train_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_train_main.cpp @@ -54,7 +54,7 @@ BINDING_EXAMPLE( // See also... BINDING_SEE_ALSO("Linear/ridge regression tutorial", - "@doxygen/lrtutorial.html"); + "@doc/tutorials/linear_regression.md"); PARAM_MATRIX_IN_REQ("training", "Matrix containing training set X (regressors).", "t"); diff --git a/src/mlpack/methods/linear_svm/linear_svm_main.cpp b/src/mlpack/methods/linear_svm/linear_svm_main.cpp index dccb7ae66f..b663f29666 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_main.cpp +++ b/src/mlpack/methods/linear_svm/linear_svm_main.cpp @@ -109,7 +109,7 @@ BINDING_SEE_ALSO("@logistic_regression", "#logistic_regression"); BINDING_SEE_ALSO("LinearSVM on Wikipedia", "https://en.wikipedia.org/wiki/Support-vector_machine"); BINDING_SEE_ALSO("mlpack::svm::LinearSVM C++ class documentation", - "@doxygen/classmlpack_1_1svm_1_1LinearSVM.html"); + "@src/mlpack/methods/linear_svm/linear_svm.hpp"); // Training parameters. PARAM_MATRIX_IN("training", "A matrix containing the training set (the matrix " diff --git a/src/mlpack/methods/lmnn/lmnn_main.cpp b/src/mlpack/methods/lmnn/lmnn_main.cpp index 7ab8c4798e..7187da4ee3 100644 --- a/src/mlpack/methods/lmnn/lmnn_main.cpp +++ b/src/mlpack/methods/lmnn/lmnn_main.cpp @@ -138,7 +138,7 @@ BINDING_SEE_ALSO("Distance metric learning for large margin nearest neighbor " "classification (pdf)", "http://papers.nips.cc/paper/2795-distance-" "metric-learning-for-large-margin-nearest-neighbor-classification.pdf"); BINDING_SEE_ALSO("mlpack::lmnn::LMNN C++ class documentation", - "@doxygen/classmlpack_1_1lmnn_1_1LMNN.html"); + "@src/mlpack/methods/lmnn/lmnn.hpp"); PARAM_MATRIX_IN_REQ("input", "Input dataset to run LMNN on.", "i"); PARAM_MATRIX_IN("distance", "Initial distance matrix to be used as " diff --git a/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp b/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp index 5adb94a1fe..2a44abfd2b 100644 --- a/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp +++ b/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp @@ -93,7 +93,8 @@ BINDING_SEE_ALSO("Nonlinear learning using local coordinate coding (pdf)", "https://papers.nips.cc/paper/3875-nonlinear-learning-using-local-" "coordinate-coding.pdf"); BINDING_SEE_ALSO("mlpack::lcc::LocalCoordinateCoding C++ class documentation", - "@doxygen/classmlpack_1_1lcc_1_1LocalCoordinateCoding.html"); + "@src/mlpack/methods/local_coordinate_coding/local_coordinate_coding." + "hpp"); // Training parameters. PARAM_MATRIX_IN("training", "Matrix of training data (X).", "t"); diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp index 81b9624a90..3397939c76 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp @@ -118,7 +118,7 @@ BINDING_SEE_ALSO("Logistic regression on Wikipedia", "https://en.wikipedia.org/wiki/Logistic_regression"); BINDING_SEE_ALSO("mlpack::regression::LogisticRegression C++ class " "documentation", - "@doxygen/classmlpack_1_1regression_1_1LogisticRegression.html"); + "@src/mlpack/methods/logistic_regression/logistic_regression.hpp"); // Training parameters. PARAM_MATRIX_IN("training", "A matrix containing the training set (the matrix " diff --git a/src/mlpack/methods/lsh/lsh_main.cpp b/src/mlpack/methods/lsh/lsh_main.cpp index 14f1da9175..f843f7bcb0 100644 --- a/src/mlpack/methods/lsh/lsh_main.cpp +++ b/src/mlpack/methods/lsh/lsh_main.cpp @@ -74,7 +74,7 @@ BINDING_SEE_ALSO("Locality-sensitive hashing on Wikipedia", BINDING_SEE_ALSO("Locality-sensitive hashing scheme based on p-stable" " distributions(pdf)", "http://mlpack.org/papers/lsh.pdf"); BINDING_SEE_ALSO("mlpack::neighbor::LSHSearch C++ class documentation", - "@doxygen/classmlpack_1_1neighbor_1_1LSHSearch.html"); + "@src/mlpack/methods/lsh/lsh.hpp"); // Define our input parameters that this program will take. PARAM_MATRIX_IN("reference", "Matrix containing the reference dataset.", "r"); diff --git a/src/mlpack/methods/mean_shift/mean_shift_main.cpp b/src/mlpack/methods/mean_shift/mean_shift_main.cpp index 6893dcf6a4..ce75b006e2 100644 --- a/src/mlpack/methods/mean_shift/mean_shift_main.cpp +++ b/src/mlpack/methods/mean_shift/mean_shift_main.cpp @@ -67,7 +67,7 @@ BINDING_SEE_ALSO("Mean Shift, Mode Seeking, and Clustering (pdf)", "http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.510.1222" "&rep=rep1&type=pdf"); BINDING_SEE_ALSO("mlpack::mean_shift::MeanShift C++ class documentation", - "@doxygen/classmlpack_1_1meanshift_1_1MeanShift.html"); + "@src/mlpack/methods/mean_shift/mean_shift.hpp"); // Required options. PARAM_MATRIX_IN_REQ("input", "Input dataset to perform clustering on.", "i"); diff --git a/src/mlpack/methods/naive_bayes/nbc_main.cpp b/src/mlpack/methods/naive_bayes/nbc_main.cpp index dd00f1d97f..2c3a279a48 100644 --- a/src/mlpack/methods/naive_bayes/nbc_main.cpp +++ b/src/mlpack/methods/naive_bayes/nbc_main.cpp @@ -93,7 +93,8 @@ BINDING_SEE_ALSO("@random_forest", "#random_forest"); BINDING_SEE_ALSO("Naive Bayes classifier on Wikipedia", "https://en.wikipedia.org/wiki/Naive_Bayes_classifier"); BINDING_SEE_ALSO("mlpack::naive_bayes::NaiveBayesClassifier C++ class " - "documentation", "@doxygen/classmlpack_1_1naive__bayes_1_1" + "documentation", + "@src/mlpack/methods/naive_bayes/naive_bayes_classifier.cpp" "NaiveBayesClassifier.html"); // A struct for saving the model with mappings. diff --git a/src/mlpack/methods/nca/nca_main.cpp b/src/mlpack/methods/nca/nca_main.cpp index d8d039d7ba..2dca933d91 100644 --- a/src/mlpack/methods/nca/nca_main.cpp +++ b/src/mlpack/methods/nca/nca_main.cpp @@ -100,7 +100,7 @@ BINDING_SEE_ALSO("Neighbourhood components analysis (pdf)", "http://papers.nips.cc/paper/2566-neighbourhood-components-" "analysis.pdf"); BINDING_SEE_ALSO("mlpack::nca::NCA C++ class documentation", - "@doxygen/classmlpack_1_1nca_1_1NCA.html"); + "@src/mlpack/methods/nca/nca.hpp"); PARAM_MATRIX_IN_REQ("input", "Input dataset to run NCA on.", "i"); PARAM_MATRIX_OUT("output", "Output matrix for learned distance matrix.", "o"); diff --git a/src/mlpack/methods/neighbor_search/kfn_main.cpp b/src/mlpack/methods/neighbor_search/kfn_main.cpp index f8e562c312..4c181c153b 100644 --- a/src/mlpack/methods/neighbor_search/kfn_main.cpp +++ b/src/mlpack/methods/neighbor_search/kfn_main.cpp @@ -71,7 +71,7 @@ BINDING_SEE_ALSO("@knn", "#knn"); BINDING_SEE_ALSO("Tree-independent dual-tree algorithms (pdf)", "http://proceedings.mlr.press/v28/curtin13.pdf"); BINDING_SEE_ALSO("mlpack::neighbor::NeighborSearch C++ class documentation", - "@doxygen/classmlpack_1_1neighbor_1_1NeighborSearch.html"); + "@src/mlpack/methods/neighbor_search/neighbor_search.hpp"); // Define our input parameters that this program will take. PARAM_MATRIX_IN("reference", "Matrix containing the reference dataset.", "r"); diff --git a/src/mlpack/methods/neighbor_search/knn_main.cpp b/src/mlpack/methods/neighbor_search/knn_main.cpp index 0679e6375c..187a7ecb36 100644 --- a/src/mlpack/methods/neighbor_search/knn_main.cpp +++ b/src/mlpack/methods/neighbor_search/knn_main.cpp @@ -71,11 +71,11 @@ BINDING_SEE_ALSO("@lsh", "#lsh"); BINDING_SEE_ALSO("@krann", "#krann"); BINDING_SEE_ALSO("@kfn", "#kfn"); BINDING_SEE_ALSO("NeighborSearch tutorial (k-nearest-neighbors)", - "@doxygen/nstutorial.html"); + "@doc/tutorials/neighbor_search.md"); BINDING_SEE_ALSO("Tree-independent dual-tree algorithms (pdf)", "http://proceedings.mlr.press/v28/curtin13.pdf"); BINDING_SEE_ALSO("mlpack::neighbor::NeighborSearch C++ class documentation", - "@doxygen/classmlpack_1_1neighbor_1_1NeighborSearch.html"); + "@src/mlpack/methods/neighbor_search/neighbor_search.hpp"); // Define our input parameters that this program will take. PARAM_MATRIX_IN("reference", "Matrix containing the reference dataset.", "r"); diff --git a/src/mlpack/methods/nmf/nmf_main.cpp b/src/mlpack/methods/nmf/nmf_main.cpp index 749b44138b..d07a4adfc5 100644 --- a/src/mlpack/methods/nmf/nmf_main.cpp +++ b/src/mlpack/methods/nmf/nmf_main.cpp @@ -73,14 +73,14 @@ BINDING_EXAMPLE( // See also... BINDING_SEE_ALSO("@cf", "#cf"); BINDING_SEE_ALSO("Alternating matrix factorization tutorial", - "@doxygen/amftutorial.html"); + "@doc/tutorials/amf.md"); BINDING_SEE_ALSO("Non-negative matrix factorization on Wikipedia", "https://en.wikipedia.org/wiki/Non-negative_matrix_factorization"); BINDING_SEE_ALSO("Algorithms for non-negative matrix factorization (pdf)", "http://papers.nips.cc/paper/1861-algorithms-for-non-negative-matrix-" "factorization.pdf"); BINDING_SEE_ALSO("mlpack::amf::AMF C++ class documentation", - "@doxygen/classmlpack_1_1amf_1_1AMF.html"); + "@src/mlpack/methods/amf/amf.hpp"); // Parameters for program. PARAM_MATRIX_IN_REQ("input", "Input dataset to perform NMF on.", "i"); diff --git a/src/mlpack/methods/pca/pca_main.cpp b/src/mlpack/methods/pca/pca_main.cpp index d8f2c01458..0694516b49 100644 --- a/src/mlpack/methods/pca/pca_main.cpp +++ b/src/mlpack/methods/pca/pca_main.cpp @@ -69,7 +69,7 @@ BINDING_EXAMPLE( BINDING_SEE_ALSO("Principal component analysis on Wikipedia", "https://en.wikipedia.org/wiki/Principal_component_analysis"); BINDING_SEE_ALSO("mlpack::pca::PCA C++ class documentation", - "@doxygen/classmlpack_1_1pca_1_1PCA.html"); + "@src/mlpack/methods/pca/pca.hpp"); // Parameters for program. PARAM_MATRIX_IN_REQ("input", "Input dataset to perform PCA on.", "i"); diff --git a/src/mlpack/methods/perceptron/perceptron_main.cpp b/src/mlpack/methods/perceptron/perceptron_main.cpp index a7849780f2..02e4ada60c 100644 --- a/src/mlpack/methods/perceptron/perceptron_main.cpp +++ b/src/mlpack/methods/perceptron/perceptron_main.cpp @@ -105,7 +105,7 @@ BINDING_SEE_ALSO("@adaboost", "#adaboost"); BINDING_SEE_ALSO("Perceptron on Wikipedia", "https://en.wikipedia.org/wiki/Perceptron"); BINDING_SEE_ALSO("mlpack::perceptron::Perceptron C++ class documentation", - "@doxygen/classmlpack_1_1perceptron_1_1Perceptron.html"); + "@src/mlpack/methods/perceptron/perceptron.hpp"); // When we save a model, we must also save the class mappings. So we use this // auxiliary structure to store both the perceptron and the mapping, and we'll diff --git a/src/mlpack/methods/radical/radical_main.cpp b/src/mlpack/methods/radical/radical_main.cpp index 3a4c6730a8..759fbc0fbf 100644 --- a/src/mlpack/methods/radical/radical_main.cpp +++ b/src/mlpack/methods/radical/radical_main.cpp @@ -57,7 +57,7 @@ BINDING_SEE_ALSO("ICA using spacings estimates of entropy (pdf)", "http://www.jmlr.org/papers/volume4/learned-miller03a/" "learned-miller03a.pdf"); BINDING_SEE_ALSO("mlpack::radical::Radical C++ class documentation", - "@doxygen/classmlpack_1_1radical_1_1Radical.html"); + "@src/mlpack/methods/radical/radical.hpp"); PARAM_MATRIX_IN_REQ("input", "Input dataset for ICA.", "i"); diff --git a/src/mlpack/methods/random_forest/random_forest_main.cpp b/src/mlpack/methods/random_forest/random_forest_main.cpp index a71d131ed8..936bbe5e2a 100644 --- a/src/mlpack/methods/random_forest/random_forest_main.cpp +++ b/src/mlpack/methods/random_forest/random_forest_main.cpp @@ -105,7 +105,7 @@ BINDING_SEE_ALSO("Random forest on Wikipedia", BINDING_SEE_ALSO("Random forests (pdf)", "https://link.springer.com/content/pdf/10.1023/A:1010933404324.pdf"); BINDING_SEE_ALSO("mlpack::tree::RandomForest C++ class documentation", - "@doxygen/classmlpack_1_1tree_1_1RandomForest.html"); + "@src/mlpack/methods/random_forest/random_forest.cpp"); PARAM_MATRIX_IN("training", "Training dataset.", "t"); PARAM_UROW_IN("labels", "Labels for training dataset.", "l"); diff --git a/src/mlpack/methods/range_search/range_search_main.cpp b/src/mlpack/methods/range_search/range_search_main.cpp index 8dce4b1da0..2c9ed69a73 100644 --- a/src/mlpack/methods/range_search/range_search_main.cpp +++ b/src/mlpack/methods/range_search/range_search_main.cpp @@ -75,12 +75,13 @@ BINDING_EXAMPLE( // See also... BINDING_SEE_ALSO("@knn", "#knn"); +BINDING_SEE_ALSO("Range search tutorial", "@doc/tutorials/range_search.md"); BINDING_SEE_ALSO("Range searching on Wikipedia", "https://en.wikipedia.org/wiki/Range_searching"); BINDING_SEE_ALSO("Tree-independent dual-tree algorithms (pdf)", "http://proceedings.mlr.press/v28/curtin13.pdf"); BINDING_SEE_ALSO("mlpack::range::RangeSearch C++ class documentation", - "@doxygen/classmlpack_1_1range_1_1RangeSearch.html"); + "@src/mlpack/methods/range_search/range_search.hpp"); // Define our input parameters that this program will take. PARAM_MATRIX_IN("reference", "Matrix containing the reference dataset.", "r"); diff --git a/src/mlpack/methods/rann/krann_main.cpp b/src/mlpack/methods/rann/krann_main.cpp index 9054fc619f..60061c817f 100644 --- a/src/mlpack/methods/rann/krann_main.cpp +++ b/src/mlpack/methods/rann/krann_main.cpp @@ -78,7 +78,7 @@ BINDING_SEE_ALSO("Rank-approximate nearest neighbor search: Retaining meaning" "3864-rank-approximate-nearest-neighbor-search-retaining-meaning-and" "-speed-in-high-dimensions.pdf"); BINDING_SEE_ALSO("mlpack::neighbor::RASearch C++ class documentation", - "@doxygen/classmlpack_1_1neighbor_1_1RASearch.html"); + "@src/mlpack/methods/rann/ra_search.hpp"); // Define our input parameters that this program will take. PARAM_MATRIX_IN("reference", "Matrix containing the reference dataset.", "r"); diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp index eb3fa36aed..b695fd0507 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp @@ -133,10 +133,6 @@ class RegularizedSVDFunction } // namespace svd } // namespace mlpack -/** - * @cond NO_DOXYGEN - */ - namespace ens { /** @@ -159,10 +155,6 @@ namespace ens { } // namespace ens -/** - * @endcond - */ - #include "regularized_svd_function_impl.hpp" #endif diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp index e4e4008152..f4506220c5 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp @@ -93,7 +93,7 @@ BINDING_SEE_ALSO("Multinomial logistic regression (softmax regression) on " "https://en.wikipedia.org/wiki/Multinomial_logistic_regression"); BINDING_SEE_ALSO("mlpack::regression::SoftmaxRegression C++ class " "documentation", - "@doxygen/classmlpack_1_1regression_1_1SoftmaxRegression.html"); + "@src/mlpack/methods/softmax_regression/softmax_regression.hpp"); // Required options. PARAM_MATRIX_IN("training", "A matrix containing the training set (the matrix " diff --git a/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp b/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp index 3801298d26..2ed43422fd 100644 --- a/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp +++ b/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp @@ -93,7 +93,7 @@ BINDING_SEE_ALSO("Regularization and variable selection via the elastic net", "http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.124.4696&" "rep=rep1&type=pdf"); BINDING_SEE_ALSO("mlpack::sparse_coding::SparseCoding C++ class documentation", - "@doxygen/classmlpack_1_1sparse__coding_1_1SparseCoding.html"); + "@src/mlpack/methods/sparse_coding/sparse_coding.hpp"); // Train the model. PARAM_MATRIX_IN("training", "Matrix of training data (X).", "t"); diff --git a/src/mlpack/methods/svdplusplus/svdplusplus_function.hpp b/src/mlpack/methods/svdplusplus/svdplusplus_function.hpp index ee0d9656cc..8f1306707c 100644 --- a/src/mlpack/methods/svdplusplus/svdplusplus_function.hpp +++ b/src/mlpack/methods/svdplusplus/svdplusplus_function.hpp @@ -147,10 +147,6 @@ class SVDPlusPlusFunction } // namespace svd } // namespace mlpack -/** - * @cond NO_DOXYGEN - */ - namespace ens { /** @@ -173,9 +169,6 @@ namespace ens { } // namespace ens -/** - * @endcond - */ #include "svdplusplus_function_impl.hpp" #endif diff --git a/src/mlpack/tests/ann/activation_functions_test.cpp b/src/mlpack/tests/ann/activation_functions_test.cpp index 4489b071e0..c3ef618e85 100644 --- a/src/mlpack/tests/ann/activation_functions_test.cpp +++ b/src/mlpack/tests/ann/activation_functions_test.cpp @@ -120,7 +120,7 @@ void CheckLeakyReLUActivationCorrect(const arma::colvec input, LeakyReLU lrf; // Test the activation function using the entire vector as input. - arma::colvec activations; + arma::colvec activations(input.n_elem); lrf.Forward(input, activations); for (size_t i = 0; i < activations.n_elem; ++i) { @@ -143,7 +143,7 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, LeakyReLU lrf; // Test the calculation of the derivatives using the entire vector as input. - arma::colvec derivatives; + arma::colvec derivatives(input.n_elem); // This error vector will be set to 1 to get the derivatives. arma::colvec error = arma::ones(input.n_elem); diff --git a/src/mlpack/tests/ann/convolutional_network_test.cpp b/src/mlpack/tests/ann/convolutional_network_test.cpp index 927ca18154..baf45d907f 100644 --- a/src/mlpack/tests/ann/convolutional_network_test.cpp +++ b/src/mlpack/tests/ann/convolutional_network_test.cpp @@ -85,29 +85,29 @@ TEST_CASE("PaddingTest", "[ConvolutionalNetworktest]") model.Forward(X, results); // Ensure that things are correctly padded. - arma::cube reshapedResults(results.memptr(), 35, 31, results.n_cols, false, + arma::cube reshapedResults(results.memptr(), 31, 35, results.n_cols, false, true); for (size_t i = 0; i < reshapedResults.n_slices; ++i) { // Check left. - for (size_t j = 0; j < reshapedResults.n_rows; ++j) - REQUIRE(reshapedResults(j, 0, i) == 0.0); + for (size_t j = 0; j < reshapedResults.n_cols; ++j) + REQUIRE(reshapedResults(0, j, i) == 0.0); // Check top. for (size_t j = 0; j < 3; ++j) - for (size_t k = 0; k < reshapedResults.n_cols; ++k) - REQUIRE(reshapedResults(j, k, i) == 0.0); + for (size_t k = 0; k < reshapedResults.n_rows; ++k) + REQUIRE(reshapedResults(k, j, i) == 0.0); // Check bottom. - for (size_t j = 31; j < reshapedResults.n_rows; ++j) - for (size_t k = 0; k < reshapedResults.n_cols; ++k) - REQUIRE(reshapedResults(j, k, i) == 0.0); + for (size_t j = 31; j < reshapedResults.n_cols; ++j) + for (size_t k = 0; k < reshapedResults.n_rows; ++k) + REQUIRE(reshapedResults(k, j, i) == 0.0); // Check right. - for (size_t j = 0; j < reshapedResults.n_rows; ++j) - for (size_t k = 29; k < reshapedResults.n_cols; ++k) - REQUIRE(reshapedResults(j, k, i) == 0.0); + for (size_t j = 0; j < reshapedResults.n_cols; ++j) + for (size_t k = 29; k < reshapedResults.n_rows; ++k) + REQUIRE(reshapedResults(k, j, i) == 0.0); } } diff --git a/src/mlpack/tests/ann/feedforward_network_test.cpp b/src/mlpack/tests/ann/feedforward_network_test.cpp index 5894fc26b8..e5d5a8085b 100644 --- a/src/mlpack/tests/ann/feedforward_network_test.cpp +++ b/src/mlpack/tests/ann/feedforward_network_test.cpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#define MLPACK_ENABLE_ANN_SERIALIZATION #include - #include #include @@ -80,8 +80,7 @@ void CheckCopyFunction(ModelType* network1, template void CheckMoveFunction(ModelType* network1, MatType& trainData, - MatType& trainLabels, - const size_t maxEpochs) + MatType& trainLabels) { ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols, -1); network1->Train(trainData, trainLabels, opt); @@ -150,7 +149,7 @@ TEST_CASE("CheckCopyMovingVanillaNetworkTest", "[FeedForwardNetworkTest]") CheckCopyFunction(model, trainData, trainLabels); // Check whether move constructor is working or not. - CheckMoveFunction(model1, trainData, trainLabels, 1); + CheckMoveFunction(model1, trainData, trainLabels); } /** @@ -184,7 +183,7 @@ TEST_CASE("CheckCopyMovingLinear3DNetworkTest", "[FeedForwardNetworkTest]") CheckCopyFunction(model, trainData, trainLabels); // Check whether move constructor is working or not. - CheckMoveFunction(model1, trainData, trainLabels, 1); + CheckMoveFunction(model1, trainData, trainLabels); } /** @@ -215,7 +214,7 @@ TEST_CASE("CheckCopyMovingNoisyLinearTest", "[FeedForwardNetworkTest]") model2->Add(); // Check whether move constructor is working or not. - CheckMoveFunction(model2, input, output, 1); + CheckMoveFunction(model2, input, output); } /** @@ -261,7 +260,7 @@ TEST_CASE("CheckCopyMovingConcatenateTest", "[FeedForwardNetworkTest]") model2->Add(); // Check whether move constructor is working or not. - CheckMoveFunction(model2, input, output, 1); + CheckMoveFunction(model2, input, output); } /** @@ -295,7 +294,7 @@ TEST_CASE("CheckCopyMovingDropoutNetworkTest", "[FeedForwardNetworkTest]") CheckCopyFunction(model, trainData, trainLabels); // Check whether move constructor is working or not. - CheckMoveFunction(model1, trainData, trainLabels, 1); + CheckMoveFunction(model1, trainData, trainLabels); } /** @@ -342,7 +341,7 @@ TEST_CASE("CheckCopyMovingVanillaNetworkTestNoBias", "[FeedForwardNetworkTest]") CheckCopyFunction<>(model, trainData, trainLabels); // Check whether move constructor is working or not. - CheckMoveFunction<>(model1, trainData, trainLabels, 1); + CheckMoveFunction<>(model1, trainData, trainLabels); } /** diff --git a/src/mlpack/tests/ann/layer/adaptive_max_pooling.cpp b/src/mlpack/tests/ann/layer/adaptive_max_pooling.cpp index 3acabad1ed..ea20856ff2 100644 --- a/src/mlpack/tests/ann/layer/adaptive_max_pooling.cpp +++ b/src/mlpack/tests/ann/layer/adaptive_max_pooling.cpp @@ -11,16 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/adaptive_mean_pooling.cpp b/src/mlpack/tests/ann/layer/adaptive_mean_pooling.cpp index 11af81406c..5ea5a69f92 100644 --- a/src/mlpack/tests/ann/layer/adaptive_mean_pooling.cpp +++ b/src/mlpack/tests/ann/layer/adaptive_mean_pooling.cpp @@ -11,16 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/add_merge.cpp b/src/mlpack/tests/ann/layer/add_merge.cpp index dbf2036462..4aab072366 100644 --- a/src/mlpack/tests/ann/layer/add_merge.cpp +++ b/src/mlpack/tests/ann/layer/add_merge.cpp @@ -11,16 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/alpha_dropout.cpp b/src/mlpack/tests/ann/layer/alpha_dropout.cpp index ffe28d1ec9..e3fce77844 100644 --- a/src/mlpack/tests/ann/layer/alpha_dropout.cpp +++ b/src/mlpack/tests/ann/layer/alpha_dropout.cpp @@ -11,9 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/batch_norm.cpp b/src/mlpack/tests/ann/layer/batch_norm.cpp index 5790e7eafc..ff51b600f7 100644 --- a/src/mlpack/tests/ann/layer/batch_norm.cpp +++ b/src/mlpack/tests/ann/layer/batch_norm.cpp @@ -10,17 +10,9 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#define MLPACK_ENABLE_ANN_SERIALIZATION #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/convolution.cpp b/src/mlpack/tests/ann/layer/convolution.cpp index 7c6be35bfc..1a39317c32 100644 --- a/src/mlpack/tests/ann/layer/convolution.cpp +++ b/src/mlpack/tests/ann/layer/convolution.cpp @@ -11,11 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" @@ -160,7 +156,48 @@ TEST_CASE("GradientConvolutionLayerTest", "[ANNLayerTest]") arma::mat input, target; } function; - REQUIRE(CheckGradient(function) < 1e3); + REQUIRE(CheckGradient(function) < 1e-1); +} + +/** + * Convolution layer numerical gradient test with stride = 2. + */ +TEST_CASE("GradientConvolutionLayerWithStrideTest", "[ANNLayerTest]") +{ + struct GradientFunction + { + GradientFunction() : + input(arma::linspace(0, 35, 36)), + target(arma::mat("1")) + { + model = new FFN(); + model->ResetData(input, target); + model->Add(1, 3, 3, 2, 2, std::tuple(0, 0), + std::tuple(0, 0), "same"); + model->Add(); + + model->InputDimensions() = std::vector({ 6, 6 }); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) < 1e-1); } TEST_CASE("ConvolutionLayerTestCase", "[ANNLayerTest]") @@ -327,3 +364,81 @@ TEST_CASE("AdvancedConvolutionLayerTest", "[ANNLayerTest]") layer.Backward(input, output, delta); REQUIRE(arma::accu(delta) == Approx(-1.9237523079).epsilon(1e-5)); } + +/** + * Advanced test for the Convolution layer with stride = 2. + */ +TEST_CASE("AdvancedConvolutionLayerWithStrideTest", "[ANNLayerTest]") +{ + arma::mat input, output; + + // The input test matrix is of the form 3 x 2 x 2 x 2 where + // number of images are 3 and number of feature maps are 2. + input = { { 1, 446, 42 }, + { 2, 16, 63 }, + { 1, 446, 42 }, + { 2, 16, 63 }, + { 3, 13, 63 }, + { 4, 21, 21 }, + { 3, 13, 63 }, + { 4, 21, 21 }, + { 1, 446, 42 }, + { 2, 16, 63 }, + { 1, 446, 42 }, + { 2, 16, 63 }, + { 3, 13, 63 }, + { 4, 21, 21 }, + { 3, 13, 63 }, + { 4, 21, 21 }, + { 1, 13, 11 }, + { 32, 45, 42 }, + { 1, 13, 11 }, + { 32, 45, 42 }, + { 22, 16 , 63 }, + { 32, 13 , 42 }, + { 22, 16 , 63 }, + { 32, 13 , 42 }, + { 1, 13, 11 }, + { 32, 45, 42 }, + { 1, 13, 11 }, + { 32, 45, 42 }, + { 22, 16 , 63 }, + { 32, 13 , 42 }, + { 22, 16 , 63 }, + { 32, 13 , 42 } }; + + Convolution layer(2, 2, 2, 2, 2, 0, 0); + layer.InputDimensions() = std::vector({ 4, 4, 2 }); + layer.ComputeOutputDimensions(); + arma::mat layerWeights(layer.WeightSize(), 1); + layerWeights(0) = 0.34526727; + layerWeights(1) = 0.10398731; + layerWeights(2) = -0.23198915; + layerWeights(3) = 0.05350551; + layerWeights(4) = -0.2239646; + layerWeights(5) = 0.30852968; + layerWeights(6) = -0.2635072; + layerWeights(7) = 0.01724506; + layerWeights(8) = -0.20932047; + layerWeights(9) = 0.2990749; + layerWeights(10) = -0.2981235; + layerWeights(11) = -0.14024211; + layerWeights(12) = -0.09744886; + layerWeights(13) = 0.16249102; + layerWeights(14) = 0.2692932; + layerWeights(15) = -0.12563613; + layerWeights(16) = -0.1114468053; + layerWeights(17) = -0.3029643595; + layer.SetWeights(layerWeights.memptr()); + output.set_size(layer.OutputSize(), 3); + + layer.Forward(input, output); + + // Value calculated using torch.nn.Conv2d(). + REQUIRE(arma::accu(output) == Approx(364.7379150391).epsilon(1e-5)); + + arma::mat delta; + delta.set_size(32, 3); + layer.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == Approx(115.3515701294).epsilon(1e-5)); +} diff --git a/src/mlpack/tests/ann/layer/dropout.cpp b/src/mlpack/tests/ann/layer/dropout.cpp index 3da290502f..31389dd98f 100644 --- a/src/mlpack/tests/ann/layer/dropout.cpp +++ b/src/mlpack/tests/ann/layer/dropout.cpp @@ -11,9 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/grouped_convolution.cpp b/src/mlpack/tests/ann/layer/grouped_convolution.cpp index 21c3f99b44..cdb61a472f 100644 --- a/src/mlpack/tests/ann/layer/grouped_convolution.cpp +++ b/src/mlpack/tests/ann/layer/grouped_convolution.cpp @@ -10,11 +10,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/identity.cpp b/src/mlpack/tests/ann/layer/identity.cpp index 9653b8d8e2..fe45f380ca 100644 --- a/src/mlpack/tests/ann/layer/identity.cpp +++ b/src/mlpack/tests/ann/layer/identity.cpp @@ -11,16 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/linear3d.cpp b/src/mlpack/tests/ann/layer/linear3d.cpp index a8f73b21ce..ac2771e9ac 100644 --- a/src/mlpack/tests/ann/layer/linear3d.cpp +++ b/src/mlpack/tests/ann/layer/linear3d.cpp @@ -11,12 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/linear_no_bias.cpp b/src/mlpack/tests/ann/layer/linear_no_bias.cpp index 5a56e37cb8..e446e86bb7 100644 --- a/src/mlpack/tests/ann/layer/linear_no_bias.cpp +++ b/src/mlpack/tests/ann/layer/linear_no_bias.cpp @@ -11,11 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/log_softmax.cpp b/src/mlpack/tests/ann/layer/log_softmax.cpp index 2686c90eff..d6b14dbab0 100644 --- a/src/mlpack/tests/ann/layer/log_softmax.cpp +++ b/src/mlpack/tests/ann/layer/log_softmax.cpp @@ -11,11 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/max_pooling.cpp b/src/mlpack/tests/ann/layer/max_pooling.cpp index 300181a28e..ecadac8108 100644 --- a/src/mlpack/tests/ann/layer/max_pooling.cpp +++ b/src/mlpack/tests/ann/layer/max_pooling.cpp @@ -11,10 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/mean_pooling.cpp b/src/mlpack/tests/ann/layer/mean_pooling.cpp index f648f9ef13..341d47f24d 100644 --- a/src/mlpack/tests/ann/layer/mean_pooling.cpp +++ b/src/mlpack/tests/ann/layer/mean_pooling.cpp @@ -11,16 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" diff --git a/src/mlpack/tests/ann/layer/padding.cpp b/src/mlpack/tests/ann/layer/padding.cpp index 2a25baed50..c546b2f05e 100644 --- a/src/mlpack/tests/ann/layer/padding.cpp +++ b/src/mlpack/tests/ann/layer/padding.cpp @@ -11,10 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp" @@ -43,7 +40,7 @@ TEST_CASE("SimplePaddingLayerTest", "[ANNLayerTest]") output.randu(); module.Forward(input, output); REQUIRE(arma::accu(input) == Approx(arma::accu(output))); - REQUIRE(output.n_rows == (9 * 8)); // 2x5 --> 9x8 + REQUIRE(output.n_rows == (5 * 12)); // 2x5 --> 5x12 // Test the Backward function. delta.set_size(input.n_rows, input.n_cols); diff --git a/src/mlpack/tests/ann/layer/softmax.cpp b/src/mlpack/tests/ann/layer/softmax.cpp index afbf517112..d013a48f63 100644 --- a/src/mlpack/tests/ann/layer/softmax.cpp +++ b/src/mlpack/tests/ann/layer/softmax.cpp @@ -11,16 +11,7 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "../../test_catch_tools.hpp" #include "../../catch.hpp"