Merge remote-tracking branch 'upstream/master'

.appveyor.yml

@@ -30,10 +30,10 @@ build_script:
   - if not exist armadillo.tar.xz appveyor DownloadFile "http://sourceforge.net/projects/arma/files/armadillo-7.800.2.tar.xz" -FileName armadillo.tar.xz
   - 7z x armadillo.tar.xz -so | 7z x -si -ttar > nul
   - cd armadillo-7.800.2 && mkdir build && cd build
-  - cmake -G "Visual Studio 14 2015 Win64" -DBLAS_LIBRARY:FILEPATH="%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARY:FILEPATH="%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DCMAKE_PREFIX:FILEPATH="%APPVEYOR_BUILD_FOLDER%/armadillo" -DBUILD_SHARED_LIBS=OFF ..
+  - cmake -G "Visual Studio 14 2015 Win64" -DBLAS_LIBRARY:FILEPATH="%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARY:FILEPATH="%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DCMAKE_PREFIX:FILEPATH="%APPVEYOR_BUILD_FOLDER%/armadillo" -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release ..
   - '"C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" "C:\projects\mlpack\armadillo-7.800.2\build\armadillo.sln" /m /verbosity:quiet /p:Configuration=Release;Platform=x64'
   - cd C:\projects\mlpack && mkdir build && cd build
-  - cmake -G "Visual Studio 14 2015 Win64" -DBLAS_LIBRARY:FILEPATH="%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARY:FILEPATH="%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DARMADILLO_INCLUDE_DIR="C:/projects/mlpack/armadillo-7.800.2/include" -DARMADILLO_LIBRARY:FILEPATH="C:\projects\mlpack\armadillo-7.800.2\build\Debug\armadillo.lib" -DBOOST_INCLUDEDIR:PATH="C:\projects\mlpack\boost.1.60.0.0\lib\native\include" -DBOOST_LIBRARYDIR:PATH="C:\projects\mlpack\boost_libs" -DDEBUG=ON -DPROFILE=ON -DBUILD_PYTHON_BINDINGS=OFF ..
+  - cmake -G "Visual Studio 14 2015 Win64" -DBLAS_LIBRARY:FILEPATH="%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DLAPACK_LIBRARY:FILEPATH="%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" -DARMADILLO_INCLUDE_DIR="C:/projects/mlpack/armadillo-7.800.2/include" -DARMADILLO_LIBRARY:FILEPATH="C:\projects\mlpack\armadillo-7.800.2\build\Debug\armadillo.lib" -DBOOST_INCLUDEDIR:PATH="C:\projects\mlpack\boost.1.60.0.0\lib\native\include" -DBOOST_LIBRARYDIR:PATH="C:\projects\mlpack\boost_libs" -DDEBUG=OFF -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF -DCMAKE_BUILD_TYPE=Release ..
   - '"C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" "C:\projects\mlpack\build\mlpack.sln" /m /verbosity:minimal /nologo /p:BuildInParallel=true /p:Configuration=Release;Platform=x64'
   - 7z a mlpack-windows-no-libs.zip "%APPVEYOR_BUILD_FOLDER%\build\Release\*.exe"
   - 7z a mlpack-windows.zip "%APPVEYOR_BUILD_FOLDER%\build\Release\*.*" "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/*.*"
@@ -53,8 +53,13 @@ cache:
   - packages -> **\packages.config
   - armadillo.tar.xz -> appveyor.yaml
 
-# All plans have maximum build job execution time of 60 minutes. But right, now
-# the machine takes 30 minutes to build the code and at least 50 minutes to run
-# all tests.
-# test_script:   -
-# '"C:\projects\mlpack\build\Release\mlpack_test.exe" -p'
+test_script:
+  # Copy all DLLs into the right place before running the test.
+  - ps: cp C:\projects\mlpack\boost_libs\*.* C:\projects\mlpack\build\
+  - ps: cp C:\projects\mlpack\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* C:\projects\mlpack\build\
+  - cd "%APPVEYOR_BUILD_FOLDER%/build/"
+  - Release\mlpack_test.exe --report_level=detailed --log_level=test_suite --log_format=XML > mlpack_test.xml & exit 0
+  # Attempt to upload results to AppVeyor.
+  - ps: |
+        $wc = New-Object 'System.Net.WebClient'
+        $wc.UploadFile("https://ci.appveyor.com/api/testresults/xunit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\mlpack_test.xml))

CMakeLists.txt

@@ -64,6 +64,11 @@ endif()
 # specific.  This list is a subset of MLPACK_LIBRARIES.
 set(COMPILER_SUPPORT_LIBRARIES "")
 
+# If we are using MSVC, we need /bigobj.
+if (MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
+endif ()
+
 # If using clang, we have to link against libc++ depending on the
 # OS (at least on some systems). Further, gcc sometimes optimizes calls to
 # math.h functions, making -lm unnecessary with gcc, but it may still be
@@ -142,8 +147,6 @@ if(DEBUG)
     add_definitions(-DDEBUG)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -ftemplate-backtrace-limit=0")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -g -O0")
-  else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
   endif()
 
   # mlpack uses it's own mlpack::backtrace class based on Binary File Descriptor

COPYRIGHT.txt

@@ -87,6 +87,7 @@ Copyright:
   Copyright 2017, Manish Kumar <manish887kr@gmail.com>
   Copyright 2017, Haritha Sreedharan Nair <haritha1313@gmail.com>
   Copyright 2017&2018, Sourabh Varshney <sourabhvarshney111@gmail.com>
+  Copyright 2018, Nikhil Goel <nikhilgoel199797@gmail.com>
 
 License: BSD-3-clause
   All rights reserved.

src/mlpack/bindings/cli/CMakeLists.txt

@@ -5,7 +5,9 @@ set(SOURCES
   cli_option.hpp
   default_param.hpp
   default_param_impl.hpp
+  delete_allocated_memory.hpp
   end_program.hpp
+  get_allocated_memory.hpp
   get_param.hpp
   get_raw_param.hpp
   get_printable_param.hpp

src/mlpack/bindings/cli/add_to_po.hpp

@@ -89,13 +89,15 @@ void AddToPO(const util::ParamData& d,
       (boost::program_options::options_description*) output;
 
   // Generate the name to be given to boost::program_options.
-  const std::string mappedName = MapParameterName<T>(d.name);
+  const std::string mappedName =
+      MapParameterName<typename std::remove_pointer<T>::type>(d.name);
   std::string boostName = (d.alias != '\0') ? mappedName + "," +
       std::string(1, d.alias) : mappedName;
 
   // Note that we have to add the option as type equal to the mapped type, not
   // the true type of the option.
-  AddToPO<typename ParameterType<T>::type>(boostName, d.desc, *desc);
+  AddToPO<typename ParameterType<typename std::remove_pointer<T>::type>::type>(
+      boostName, d.desc, *desc);
 }
 
 } // namespace cli

src/mlpack/bindings/cli/cli_option.hpp

@@ -28,6 +28,8 @@
 #include "set_param.hpp"
 #include "get_printable_param_name.hpp"
 #include "get_printable_param_value.hpp"
+#include "get_allocated_memory.hpp"
+#include "delete_allocated_memory.hpp"
 
 namespace mlpack {
 namespace bindings {
@@ -88,19 +90,21 @@ class CLIOption
     data.cppType = cppName;
 
     // Apply default value.
-    if (std::is_same<N, typename ParameterType<N>::type>::value)
+    if (std::is_same<typename std::remove_pointer<N>::type,
+                     typename ParameterType<typename
+                         std::remove_pointer<N>::type>::type>::value)
     {
       data.value = boost::any(defaultValue);
     }
     else
     {
-      typename ParameterType<N>::type tmp;
-      data.value = boost::any(std::tuple<N, typename ParameterType<N>::type>(
-          defaultValue, tmp));
+      typename ParameterType<typename std::remove_pointer<N>::type>::type tmp;
+      data.value = boost::any(std::tuple<N, decltype(tmp)>(defaultValue, tmp));
     }
 
     const std::string tname = data.tname;
-    const std::string boostName = MapParameterName<N>(identifier);
+    const std::string boostName = MapParameterName<
+        typename std::remove_pointer<N>::type>(identifier);
     std::string progOptId = (alias[0] != '\0') ? boostName + ","
         + std::string(1, alias[0]) : boostName;
 
@@ -152,6 +156,10 @@ class CLIOption
         &GetPrintableParamName<N>;
     CLI::GetSingleton().functionMap[tname]["GetPrintableParamValue"] =
         &GetPrintableParamValue<N>;
+    CLI::GetSingleton().functionMap[tname]["GetAllocatedMemory"] =
+        &GetAllocatedMemory<N>;
+    CLI::GetSingleton().functionMap[tname]["DeleteAllocatedMemory"] =
+        &DeleteAllocatedMemory<N>;
   }
 };
 

src/mlpack/bindings/cli/default_param.hpp

@@ -54,10 +54,19 @@ std::string DefaultParamImpl(
     const util::ParamData& data,
     const typename boost::enable_if_c<
         arma::is_arma_type<T>::value ||
-        data::HasSerialize<T>::value ||
         std::is_same<T, std::tuple<mlpack::data::DatasetInfo,
                                    arma::mat>>::value>::type* /* junk */ = 0);
 
+/**
+ * Return the default value of a model option (this returns the default
+ * filename, or '' if the default is no file).
+ */
+template<typename T>
+std::string DefaultParamImpl(
+    const util::ParamData& data,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0);
+
 /**
  * Return the default value of an option.  This is the function that will be
  * placed into the CLI functionMap.
@@ -68,7 +77,7 @@ void DefaultParam(const util::ParamData& data,
                   void* output)
 {
   std::string* outstr = (std::string*) output;
-  *outstr = DefaultParamImpl<T>(data);
+  *outstr = DefaultParamImpl<typename std::remove_pointer<T>::type>(data);
 }
 
 } // namespace cli

src/mlpack/bindings/cli/default_param_impl.hpp

@@ -70,7 +70,6 @@ std::string DefaultParamImpl(
     const util::ParamData& data,
     const typename boost::enable_if_c<
         arma::is_arma_type<T>::value ||
-        data::HasSerialize<T>::value ||
         std::is_same<T, std::tuple<mlpack::data::DatasetInfo,
                                    arma::mat>>::value>::type* /* junk */)
 {
@@ -81,6 +80,24 @@ std::string DefaultParamImpl(
   return "'" + filename + "'";
 }
 
+/**
+ * Return the default value of a model option (this returns the default
+ * filename, or '' if the default is no file).
+ */
+template<typename T>
+std::string DefaultParamImpl(
+    const util::ParamData& data,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* /* junk */,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* /* junk */)
+{
+  // Get the filename and return it, or return an empty string.
+  typedef std::tuple<T*, std::string> TupleType;
+  const TupleType& tuple = *boost::any_cast<TupleType>(&data.value);
+  const std::string& filename = std::get<1>(tuple);
+  return "'" + filename + "'";
+}
+
+
 } // namespace cli
 } // namespace bindings
 } // namespace mlpack

src/mlpack/bindings/cli/delete_allocated_memory.hpp

@@ -0,0 +1,57 @@
+/**
+ * @file delete_allocated_memory.hpp
+ * @author Ryan Curtin
+ *
+ * If any memory has been allocated by the parameter, delete it.
+ */
+#ifndef MLPACK_BINDINGS_CLI_DELETE_ALLOCATED_MEMORY_HPP
+#define MLPACK_BINDINGS_CLI_DELETE_ALLOCATED_MEMORY_HPP
+
+#include <mlpack/core/util/param_data.hpp>
+
+namespace mlpack {
+namespace bindings {
+namespace cli {
+
+template<typename T>
+void DeleteAllocatedMemoryImpl(
+    const util::ParamData& /* d */,
+    const typename boost::disable_if<data::HasSerialize<T>>::type* = 0,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0)
+{
+  // Do nothing.
+}
+
+template<typename T>
+void DeleteAllocatedMemoryImpl(
+    const util::ParamData& /* d */,
+    const typename boost::enable_if<arma::is_arma_type<T>>::type* = 0)
+{
+  // Do nothing.
+}
+
+template<typename T>
+void DeleteAllocatedMemoryImpl(
+    const util::ParamData& d,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
+{
+  // Delete the allocated memory (hopefully we actually own it).
+  typedef std::tuple<T*, std::string> TupleType;
+  delete std::get<0>(*boost::any_cast<TupleType>(&d.value));
+}
+
+template<typename T>
+void DeleteAllocatedMemory(
+    const util::ParamData& d,
+    const void* /* input */,
+    void* /* output */)
+{
+  DeleteAllocatedMemoryImpl<typename std::remove_pointer<T>::type>(d);
+}
+
+} // namespace cli
+} // namespace bindings
+} // namespace mlpack
+
+#endif

src/mlpack/bindings/cli/end_program.hpp

@@ -67,6 +67,37 @@ inline void EndProgram()
       CLI::GetSingleton().timer.PrintTimer(it2.first);
     }
   }
+
+  // Lastly clean up any memory.  If we are holding any pointers, then we "own"
+  // them.  But we may hold the same pointer twice, so we have to be careful to
+  // not delete it multiple times.
+  std::unordered_map<void*, const util::ParamData*> memoryAddresses;
+  it = parameters.begin();
+  while (it != parameters.end())
+  {
+    const util::ParamData& data = it->second;
+
+    void* result;
+    CLI::GetSingleton().functionMap[data.tname]["GetAllocatedMemory"](data,
+        NULL, (void*) &result);
+    if (result != NULL && memoryAddresses.count(result) == 0)
+      memoryAddresses[result] = &data;
+
+    ++it;
+  }
+
+  // Now we have all the unique addresses that need to be deleted.
+  std::unordered_map<void*, const util::ParamData*>::const_iterator it2;
+  it2 = memoryAddresses.begin();
+  while (it2 != memoryAddresses.end())
+  {
+    const util::ParamData& data = *(it2->second);
+
+    CLI::GetSingleton().functionMap[data.tname]["DeleteAllocatedMemory"](data,
+        NULL, NULL);
+
+    ++it2;
+  }
 }
 
 } // namespace cli

src/mlpack/bindings/cli/get_allocated_memory.hpp

@@ -0,0 +1,59 @@
+/**
+ * @file get_allocated_memory.hpp
+ * @author Ryan Curtin
+ *
+ * If the parameter has a type that may need to be deleted, return the address
+ * of that object.  Otherwise return NULL.
+ */
+#ifndef MLPACK_BINDINGS_CLI_GET_ALLOCATED_MEMORY_HPP
+#define MLPACK_BINDINGS_CLI_GET_ALLOCATED_MEMORY_HPP
+
+#include <mlpack/core/util/param_data.hpp>
+
+namespace mlpack {
+namespace bindings {
+namespace cli {
+
+template<typename T>
+void* GetAllocatedMemory(
+    const util::ParamData& /* d */,
+    const typename boost::disable_if<data::HasSerialize<T>>::type* = 0,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0)
+{
+  return NULL;
+}
+
+template<typename T>
+void* GetAllocatedMemory(
+    const util::ParamData& /* d */,
+    const typename boost::enable_if<arma::is_arma_type<T>>::type* = 0)
+{
+  return NULL;
+}
+
+template<typename T>
+void* GetAllocatedMemory(
+    const util::ParamData& d,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
+{
+  // Here we have a model, which is a tuple, and we need the address of the
+  // memory.
+  typedef std::tuple<T*, std::string> TupleType;
+  return std::get<0>(*boost::any_cast<TupleType>(&d.value));
+}
+
+template<typename T>
+void GetAllocatedMemory(const util::ParamData& d,
+                        const void* /* input */,
+                        void* output)
+{
+  *((void**) output) =
+      GetAllocatedMemory<typename std::remove_pointer<T>::type>(d);
+}
+
+} // namespace cli
+} // namespace bindings
+} // namespace mlpack
+
+#endif

src/mlpack/bindings/cli/get_param.hpp

@@ -95,24 +95,24 @@ T& GetParam(
  * @param d ParamData object to get parameter value from.
  */
 template<typename T>
-T& GetParam(
+T*& GetParam(
     util::ParamData& d,
     const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
     const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
 {
   // If the model is an input model, we have to load it from file.  'value'
   // contains the filename.
-  typedef std::tuple<T, std::string> TupleType;
+  typedef std::tuple<T*, std::string> TupleType;
   TupleType* tuple = boost::any_cast<TupleType>(&d.value);
   const std::string& value = std::get<1>(*tuple);
-  T& model = std::get<0>(*tuple);
   if (d.input && !d.loaded)
   {
-    data::Load(value, "model", model, true);
+    T* model = new T();
+    data::Load(value, "model", *model, true);
     d.loaded = true;
+    std::get<0>(*tuple) = model;
   }
-
-  return model;
+  return std::get<0>(*tuple);
 }
 
 /**
@@ -127,7 +127,8 @@ template<typename T>
 void GetParam(const util::ParamData& d, const void* /* input */, void* output)
 {
   // Cast to the correct type.
-  *((T**) output) = &GetParam<T>(const_cast<util::ParamData&>(d));
+  *((T**) output) = &GetParam<typename std::remove_pointer<T>::type>(
+      const_cast<util::ParamData&>(d));
 }
 
 } // namespace cli

src/mlpack/bindings/cli/get_printable_param.hpp

@@ -37,16 +37,24 @@ std::string GetPrintableParam(
     const typename std::enable_if<util::IsStdVector<T>::value>::type* = 0);
 
 /**
- * Print a matrix option (this just prints the filename).
+ * Print a matrix/tuple option (this just prints the filename).
  */
 template<typename T>
 std::string GetPrintableParam(
     const util::ParamData& data,
     const typename std::enable_if<arma::is_arma_type<T>::value ||
-                                  data::HasSerialize<T>::value ||
                                   std::is_same<T,
         std::tuple<data::DatasetInfo, arma::mat>>::value>::type* = 0);
 
+/**
+ * Print a model option (this just prints the filename).
+ */
+template<typename T>
+std::string GetPrintableParam(
+    const util::ParamData& data,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0);
+
 /**
  * Print an option into a std::string.  This should print a short, one-line
  * representation of the object.  The string will be stored in the output
@@ -57,7 +65,8 @@ void GetPrintableParam(const util::ParamData& data,
                        const void* /* input */,
                        void* output)
 {
-  *((std::string*) output) = GetPrintableParam<T>(data);
+  *((std::string*) output) =
+      GetPrintableParam<typename std::remove_pointer<T>::type>(data);
 }
 
 } // namespace cli

src/mlpack/bindings/cli/get_printable_param_impl.hpp

@@ -43,12 +43,11 @@ std::string GetPrintableParam(
   return oss.str();
 }
 
-//! Print a matrix/model/tuple option (this just prints the filename).
+//! Print a matrix/tuple option (this just prints the filename).
 template<typename T>
 std::string GetPrintableParam(
     const util::ParamData& data,
     const typename std::enable_if<arma::is_arma_type<T>::value ||
-                                  data::HasSerialize<T>::value ||
                                   std::is_same<T,
         std::tuple<data::DatasetInfo, arma::mat>>::value>::type* /* junk */)
 {
@@ -61,6 +60,22 @@ std::string GetPrintableParam(
   return oss.str();
 }
 
+//! Print a model option (this just prints the filename).
+template<typename T>
+std::string GetPrintableParam(
+    const util::ParamData& data,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* /* junk */,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* /* junk */)
+{
+  // Extract the string from the tuple that's being held.
+  typedef std::tuple<T*, typename ParameterType<T>::type> TupleType;
+  const TupleType* tuple = boost::any_cast<TupleType>(&data.value);
+
+  std::ostringstream oss;
+  oss << std::get<1>(*tuple);
+  return oss.str();
+}
+
 } // namespace cli
 } // namespace bindings
 } // namespace mlpack

src/mlpack/bindings/cli/get_printable_param_name.hpp

@@ -64,7 +64,8 @@ void GetPrintableParamName(
     const void* /* input */,
     void* output)
 {
-  *((std::string*) output) = GetPrintableParamName<T>(d);
+  *((std::string*) output) =
+      GetPrintableParamName<typename std::remove_pointer<T>::type>(d);
 }
 
 } // namespace cli

src/mlpack/bindings/cli/get_printable_param_value.hpp

@@ -68,7 +68,8 @@ void GetPrintableParamValue(
     const void* input,
     void* output)
 {
-  *((std::string*) output) = GetPrintableParamValue<T>(d,
+  *((std::string*) output) =
+      GetPrintableParamValue<typename std::remove_pointer<T>::type>(d,
       *((std::string*) input));
 }
 

src/mlpack/bindings/cli/get_raw_param.hpp

@@ -40,15 +40,29 @@ T& GetRawParam(
     const typename boost::enable_if_c<
         arma::is_arma_type<T>::value ||
         std::is_same<T, std::tuple<mlpack::data::DatasetInfo,
-                                   arma::mat>>::value ||
-        data::HasSerialize<T>::value>::type* = 0)
+                                   arma::mat>>::value>::type* = 0)
 {
-  // Don't load the matrix/model.
+  // Don't load the matrix.
   typedef std::tuple<T, std::string> TupleType;
   T& value = std::get<0>(*boost::any_cast<TupleType>(&d.value));
   return value;
 }
 
+/**
+ * Return the name of a model parameter.
+ */
+template<typename T>
+T*& GetRawParam(
+    util::ParamData& d,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
+{
+  // Don't load the model.
+  typedef std::tuple<T*, std::string> TupleType;
+  T*& value = std::get<0>(*boost::any_cast<TupleType>(&d.value));
+  return value;
+}
+
 /**
  * Return a parameter casted to the given type.  Type checking does not happen
  * here!
@@ -63,7 +77,8 @@ void GetRawParam(const util::ParamData& d,
                  void* output)
 {
   // Cast to the correct type.
-  *((T**) output) = &GetRawParam<T>(const_cast<util::ParamData&>(d));
+  *((T**) output) = &GetRawParam<typename std::remove_pointer<T>::type>(
+      const_cast<util::ParamData&>(d));
 }
 
 } // namespace cli

src/mlpack/bindings/cli/map_parameter_name.hpp

@@ -61,7 +61,8 @@ void MapParameterName(const util::ParamData& d,
 {
   // Store the mapped name in the output pointer, which is actually a string
   // pointer.
-  *((std::string*) output) = MapParameterName<T>(d.name);
+  *((std::string*) output) =
+      MapParameterName<typename std::remove_pointer<T>::type>(d.name);
 }
 
 } // namespace cli

src/mlpack/bindings/cli/output_param.hpp

@@ -70,7 +70,7 @@ void OutputParam(const util::ParamData& data,
                  const void* /* input */,
                  void* /* output */)
 {
-  OutputParamImpl<T>(data);
+  OutputParamImpl<typename std::remove_pointer<T>::type>(data);
 }
 
 } // namespace cli

src/mlpack/bindings/cli/output_param_impl.hpp

@@ -55,10 +55,10 @@ void OutputParamImpl(
 
   if (output.n_elem > 0 && filename != "")
   {
-      if (arma::is_Row<T>::value || arma::is_Col<T>::value)
-        data::Save(filename, output, false);
-      else
-        data::Save(filename, output, false, !data.noTranspose);
+    if (arma::is_Row<T>::value || arma::is_Col<T>::value)
+      data::Save(filename, output, false);
+    else
+      data::Save(filename, output, false, !data.noTranspose);
   }
 }
 
@@ -72,14 +72,14 @@ void OutputParamImpl(
   // The const cast is necessary here because Serialize() can't ever be marked
   // const.  In this case we can assume it though, since we will be saving and
   // not loading.
-  typedef std::tuple<T, std::string> TupleType;
-  T& output = const_cast<T&>(std::get<0>(*boost::any_cast<TupleType>(
+  typedef std::tuple<T*, std::string> TupleType;
+  T*& output = const_cast<T*&>(std::get<0>(*boost::any_cast<TupleType>(
       &data.value)));
   const std::string& filename =
       std::get<1>(*boost::any_cast<TupleType>(&data.value));
 
   if (filename != "")
-    data::Save(filename, "model", output);
+    data::Save(filename, "model", *output);
 }
 
 //! Output a mapped dataset.

src/mlpack/bindings/cli/parse_command_line.hpp

@@ -52,7 +52,7 @@ void ParseCommandLine(int argc, char** argv)
     boostNameMap[boostName] = d.name;
   }
 
-  // TODO: we have to mark somehow that we parsed.
+  // Mark that we did parsing.
   CLI::GetSingleton().didParse = true;
 
   // Parse the command line, then place the values in the right place.

src/mlpack/bindings/cli/set_param.hpp

@@ -54,7 +54,6 @@ void SetParam(
     util::ParamData& d,
     const boost::any& value,
     const typename std::enable_if<arma::is_arma_type<T>::value ||
-                                  data::HasSerialize<T>::value ||
                                   std::is_same<T,
         std::tuple<data::DatasetInfo, arma::mat>>::value>::type* = 0)
 {
@@ -64,6 +63,23 @@ void SetParam(
   std::get<1>(tuple) = boost::any_cast<std::string>(value);
 }
 
+/**
+ * Set a serializable object.  This sets the filename referring to the
+ * parameter.
+ */
+template<typename T>
+void SetParam(
+    util::ParamData& d,
+    const boost::any& value,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
+{
+  // We're setting the string filename.
+  typedef std::tuple<T*, typename ParameterType<T>::type> TupleType;
+  TupleType& tuple = *boost::any_cast<TupleType>(&d.value);
+  std::get<1>(tuple) = boost::any_cast<std::string>(value);
+}
+
 /**
  * Return a parameter casted to the given type.  Type checking does not happen
  * here!
@@ -75,7 +91,8 @@ void SetParam(
 template<typename T>
 void SetParam(const util::ParamData& d, const void* input, void* /* output */)
 {
-  SetParam<T>(const_cast<util::ParamData&>(d), *((boost::any*) input));
+  SetParam<typename std::remove_pointer<T>::type>(
+      const_cast<util::ParamData&>(d), *((boost::any*) input));
 }
 
 } // namespace cli

src/mlpack/bindings/python/CMakeLists.txt

@@ -79,7 +79,6 @@ set(CYTHON_SOURCES
   mlpack/arma_util.hpp
   mlpack/cli.pxd
   mlpack/cli_util.hpp
-  mlpack/move.hpp
   mlpack/matrix_utils.py
   mlpack/serialization.hpp
   mlpack/serialization.pxd
@@ -148,7 +147,6 @@ add_custom_command(TARGET python POST_BUILD
             mlpack/arma_util.hpp
             mlpack/cli.pxd
             mlpack/cli_util.hpp
-            mlpack/move.hpp
             mlpack/matrix_utils.py
             mlpack
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/)

src/mlpack/bindings/python/get_cython_type.hpp

@@ -113,7 +113,7 @@ inline std::string GetCythonType(
     const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
     const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
 {
-  return d.cppType;
+  return d.cppType + "*";
 }
 
 } // namespace python

src/mlpack/bindings/python/get_printable_param.hpp

@@ -73,7 +73,7 @@ std::string GetPrintableParam(
     const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
 {
   std::ostringstream oss;
-  oss << data.cppType << " model";
+  oss << data.cppType << " model at " << boost::any_cast<T*>(data.value);
   return oss.str();
 }
 
@@ -110,7 +110,8 @@ void GetPrintableParam(const util::ParamData& data,
                        const void* /* input */,
                        void* output)
 {
-  *((std::string*) output) = GetPrintableParam<T>(data);
+  *((std::string*) output) =
+      GetPrintableParam<typename std::remove_pointer<T>::type>(data);
 }
 
 } // namespace python

src/mlpack/bindings/python/import_decl.hpp

@@ -79,7 +79,7 @@ void ImportDecl(const util::ParamData& d,
                 const void* indent,
                 void* /* output */)
 {
-  ImportDecl<T>(d, *((size_t*) indent));
+  ImportDecl<typename std::remove_pointer<T>::type>(d, *((size_t*) indent));
 }
 
 } // namespace python

src/mlpack/bindings/python/mlpack/arma_numpy.pxd

@@ -19,14 +19,15 @@ import numpy
 numpy.import_array()
 
 cimport arma
+from libcpp cimport bool
 
 """
 Convert a numpy ndarray to a matrix.
 """
-cdef arma.Mat[double]* numpy_to_mat_d(numpy.ndarray[numpy.double_t, ndim=2] X) \
-    except +
-cdef arma.Mat[size_t]* numpy_to_mat_s(numpy.ndarray[numpy.npy_intp, ndim=2] X) \
-    except +
+cdef arma.Mat[double]* numpy_to_mat_d(numpy.ndarray[numpy.double_t, ndim=2] X, \
+                                      bool takeOwnership) except +
+cdef arma.Mat[size_t]* numpy_to_mat_s(numpy.ndarray[numpy.npy_intp, ndim=2] X, \
+                                      bool takeOwnership) except +
 
 """
 Convert an Armadillo object to a numpy ndarray of the given type.
@@ -39,10 +40,10 @@ cdef numpy.ndarray[numpy.npy_intp, ndim=2] mat_to_numpy_s(arma.Mat[size_t]& X) \
 """
 Convert a numpy one-dimensional ndarray to a row of the given type.
 """
-cdef arma.Row[double]* numpy_to_row_d(numpy.ndarray[numpy.double_t, ndim=1] X) \
-    except +
-cdef arma.Row[size_t]* numpy_to_row_s(numpy.ndarray[numpy.npy_intp, ndim=1] X) \
-    except +
+cdef arma.Row[double]* numpy_to_row_d(numpy.ndarray[numpy.double_t, ndim=1] X, \
+                                      bool takeOwnership) except +
+cdef arma.Row[size_t]* numpy_to_row_s(numpy.ndarray[numpy.npy_intp, ndim=1] X, \
+                                      bool takeOwnership) except +
 
 """
 Convert an Armadillo row vector to a one-dimensional numpy ndarray of the
@@ -56,10 +57,10 @@ cdef numpy.ndarray[numpy.npy_intp, ndim=1] row_to_numpy_s(arma.Row[size_t]& X) \
 """
 Convert a numpy one-dimensional ndarray to a column vector of the given type.
 """
-cdef arma.Col[double]* numpy_to_col_d(numpy.ndarray[numpy.double_t, ndim=1] X) \
-    except +
-cdef arma.Col[size_t]* numpy_to_col_s(numpy.ndarray[numpy.npy_intp, ndim=1] X) \
-    except +
+cdef arma.Col[double]* numpy_to_col_d(numpy.ndarray[numpy.double_t, ndim=1] X, \
+                                      bool takeOwnership) except +
+cdef arma.Col[size_t]* numpy_to_col_s(numpy.ndarray[numpy.npy_intp, ndim=1] X, \
+                                      bool takeOwnership) except +
 
 """
 Convert an Armadillo column vector to a one-dimensional numpy ndarray of the

src/mlpack/bindings/python/mlpack/arma_numpy.pyx

@@ -24,6 +24,7 @@ import numpy
 numpy.import_array()
 
 cimport arma
+from libcpp cimport bool
 
 cdef extern from "numpy/arrayobject.h":
   void PyArray_ENABLEFLAGS(numpy.ndarray arr, int flags)
@@ -31,6 +32,7 @@ cdef extern from "numpy/arrayobject.h":
 
 cdef extern from "<mlpack/bindings/python/mlpack/arma_util.hpp>":
   void SetMemState[T](T& m, int state)
+  size_t GetMemState[T](T& m)
   double* GetMemory(arma.Mat[double]& m)
   double* GetMemory(arma.Col[double]& m)
   double* GetMemory(arma.Row[double]& m)
@@ -38,38 +40,43 @@ cdef extern from "<mlpack/bindings/python/mlpack/arma_util.hpp>":
   size_t* GetMemory(arma.Col[size_t]& m)
   size_t* GetMemory(arma.Row[size_t]& m)
 
-cdef arma.Mat[double]* numpy_to_mat_d(numpy.ndarray[numpy.double_t, ndim=2] X) \
-    except +:
+cdef arma.Mat[double]* numpy_to_mat_d(numpy.ndarray[numpy.double_t, ndim=2] X, \
+                                      bool takeOwnership) except +:
   """
-  Convert a numpy ndarray to a matrix.
+  Convert a numpy ndarray to a matrix.  The memory will still be owned by numpy.
   """
   if not (X.flags.c_contiguous or X.flags.owndata):
     # If needed, make a copy where we own the memory.
     X = X.copy(order="C")
+    takeOwnership = True
 
-  cdef arma.Mat[double]* m = new arma.Mat[double](<double*> X.data, X.shape[1], X.shape[0], False, True)
+  cdef arma.Mat[double]* m = new arma.Mat[double](<double*> X.data, X.shape[1],\
+      X.shape[0], False, False)
 
-  # Transfer ownership to the Armadillo matrix.
-  PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
-  SetMemState[arma.Mat[double]](m[0], 0)
+  # Take ownership of the memory, if we need to.
+  if takeOwnership:
+    PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
+    SetMemState[arma.Mat[double]](m[0], 0)
 
   return m
 
-cdef arma.Mat[size_t]* numpy_to_mat_s(numpy.ndarray[numpy.npy_intp, ndim=2] X) \
-    except +:
+cdef arma.Mat[size_t]* numpy_to_mat_s(numpy.ndarray[numpy.npy_intp, ndim=2] X, \
+                                      bool takeOwnership) except +:
   """
-  Convert a numpy ndarray to a matrix.
+  Convert a numpy ndarray to a matrix.  The memory will still be owned by numpy.
   """
   if not (X.flags.c_contiguous or X.flags.owndata):
     # If needed, make a copy where we own the memory.
     X = X.copy(order="C")
+    takeOwnership = True
 
   cdef arma.Mat[size_t]* m = new arma.Mat[size_t](<size_t*> X.data, X.shape[1],
-      X.shape[0], False, True)
+      X.shape[0], False, False)
 
-  # Transfer ownership to the Armadillo matrix.
-  PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
-  SetMemState[arma.Mat[size_t]](m[0], 0)
+  # Take ownership of the memory, if we need to.
+  if takeOwnership:
+    PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
+    SetMemState[arma.Mat[size_t]](m[0], 0)
 
   return m
 
@@ -85,9 +92,10 @@ cdef numpy.ndarray[numpy.double_t, ndim=2] mat_to_numpy_d(arma.Mat[double]& X) \
   cdef numpy.ndarray[numpy.double_t, ndim=2] output = \
       numpy.PyArray_SimpleNewFromData(2, &dims[0], numpy.NPY_DOUBLE, GetMemory(X))
 
-  # Transfer memory ownership.
-  SetMemState[arma.Mat[double]](X, 1)
-  PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
+  # Transfer memory ownership, if needed.
+  if GetMemState[arma.Mat[double]](X) == 0:
+    SetMemState[arma.Mat[double]](X, 1)
+    PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
 
   return output
 
@@ -103,45 +111,52 @@ cdef numpy.ndarray[numpy.npy_intp, ndim=2] mat_to_numpy_s(arma.Mat[size_t]& X) \
   cdef numpy.ndarray[numpy.npy_intp, ndim=2] output = \
       numpy.PyArray_SimpleNewFromData(2, &dims[0], numpy.NPY_INTP, GetMemory(X))
 
-  # Transfer memory ownership.
-  SetMemState[arma.Mat[size_t]](X, 1)
-  PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
+  # Transfer memory ownership, if needed.
+  if GetMemState[arma.Mat[size_t]](X) == 0:
+    SetMemState[arma.Mat[size_t]](X, 1)
+    PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
 
   return output
 
-cdef arma.Row[double]* numpy_to_row_d(numpy.ndarray[numpy.double_t, ndim=1] X) \
-    except +:
+cdef arma.Row[double]* numpy_to_row_d(numpy.ndarray[numpy.double_t, ndim=1] X, \
+                                      bool takeOwnership) except +:
   """
-  Convert a numpy one-dimensional ndarray to a row.
+  Convert a numpy one-dimensional ndarray to a row.  The memory will still be
+  owned by numpy.
   """
   if not (X.flags.c_contiguous or X.flags.owndata):
     # If needed, make a copy where we own the memory.
     X = X.copy(order="C")
+    takeOwnership = True
 
   cdef arma.Row[double]* m = new arma.Row[double](<double*> X.data, X.shape[0],
-      False, True)
+      False, False)
 
-  # Transfer ownership to the Armadillo matrix.
-  PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
-  SetMemState[arma.Row[double]](m[0], 0)
+  # Transfer memory ownership, if needed.
+  if takeOwnership:
+    PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
+    SetMemState[arma.Row[double]](m[0], 0)
 
   return m
 
-cdef arma.Row[size_t]* numpy_to_row_s(numpy.ndarray[numpy.npy_intp, ndim=1] X) \
-    except +:
+cdef arma.Row[size_t]* numpy_to_row_s(numpy.ndarray[numpy.npy_intp, ndim=1] X, \
+                                      bool takeOwnership) except +:
   """
-  Convert a numpy one-dimensional ndarray to a row.
+  Convert a numpy one-dimensional ndarray to a row.  The memory will still be
+  owned by numpy.
   """
   if not (X.flags.c_contiguous or X.flags.owndata):
     # If needed, make a copy where we own the memory.
     X = X.copy(order="C")
+    takeOwnership = True
 
   cdef arma.Row[size_t]* m = new arma.Row[size_t](<size_t*> X.data, X.shape[0],
-      False, True)
+      False, False)
 
-  # Transfer ownership to the Armadillo matrix.
-  PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
-  SetMemState[arma.Row[size_t]](m[0], 0)
+  # Transfer memory ownership, if needed.
+  if takeOwnership:
+    PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
+    SetMemState[arma.Row[size_t]](m[0], 0)
 
   return m
 
@@ -155,9 +170,10 @@ cdef numpy.ndarray[numpy.double_t, ndim=1] row_to_numpy_d(arma.Row[double]& X) \
   cdef numpy.ndarray[numpy.double_t, ndim=1] output = \
       numpy.PyArray_SimpleNewFromData(1, &dim, numpy.NPY_DOUBLE, GetMemory(X))
 
-  # Transfer memory ownership.
-  SetMemState[arma.Row[double]](X, 1)
-  PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
+  # Transfer memory ownership, if needed.
+  if GetMemState[arma.Row[double]](X) == 0:
+    SetMemState[arma.Row[double]](X, 1)
+    PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
 
   return output
 
@@ -171,45 +187,51 @@ cdef numpy.ndarray[numpy.npy_intp, ndim=1] row_to_numpy_s(arma.Row[size_t]& X) \
   cdef numpy.ndarray[numpy.npy_intp, ndim=1] output = \
       numpy.PyArray_SimpleNewFromData(1, &dim, numpy.NPY_INTP, GetMemory(X))
 
-  # Transfer memory ownership.
-  SetMemState[arma.Row[size_t]](X, 1)
-  PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
+  # Transfer memory ownership, if needed.
+  if GetMemState[arma.Row[size_t]](X) == 0:
+    SetMemState[arma.Row[size_t]](X, 1)
+    PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
 
   return output
 
-cdef arma.Col[double]* numpy_to_col_d(numpy.ndarray[numpy.double_t, ndim=1] X) \
-    except +:
+cdef arma.Col[double]* numpy_to_col_d(numpy.ndarray[numpy.double_t, ndim=1] X, \
+                                      bool takeOwnership) except +:
   """
-  Convert a numpy one-dimensional ndarray to a column vector.
+  Convert a numpy one-dimensional ndarray to a column vector.  The memory will
+  still be owned by numpy.
   """
   if not (X.flags.c_contiguous or X.flags.owndata):
     # If needed, make a copy where we own the memory.
     X = X.copy(order="C")
+    takeOwnership = True
 
   cdef arma.Col[double]* m = new arma.Col[double](<double*> X.data, X.shape[0],
       False, True)
 
-  # Transfer ownership to the Armadillo matrix.
-  PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
-  SetMemState[arma.Col[double]](m[0], 0)
+  # Transfer memory ownership, if needed.
+  if takeOwnership:
+    PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
+    SetMemState[arma.Col[double]](m[0], 0)
 
   return m
 
-cdef arma.Col[size_t]* numpy_to_col_s(numpy.ndarray[numpy.npy_intp, ndim=1] X) \
-    except +:
+cdef arma.Col[size_t]* numpy_to_col_s(numpy.ndarray[numpy.npy_intp, ndim=1] X, \
+                                      bool takeOwnership) except +:
   """
-  Convert a numpy one-dimensional ndarray to a column vector.
+  Convert a numpy one-dimensional ndarray to a column vector.  The memory will
+  still be owned by numpy.
   """
   if not (X.flags.c_contiguous or X.flags.owndata):
     # If needed, make a copy where we own the memory.
     X = X.copy(order="C")
 
   cdef arma.Col[size_t]* m = new arma.Col[size_t](<size_t*> X.data, X.shape[0],
-      False, True)
+      False, False)
 
-  # Transfer ownership to the Armadillo matrix.
-  PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
-  SetMemState[arma.Col[size_t]](m[0], 0)
+  # Transfer memory ownership, if needed.
+  if takeOwnership:
+    PyArray_CLEARFLAGS(X, numpy.NPY_OWNDATA)
+    SetMemState[arma.Col[size_t]](m[0], 0)
 
   return m
 
@@ -223,9 +245,10 @@ cdef numpy.ndarray[numpy.double_t, ndim=1] col_to_numpy_d(arma.Col[double]& X) \
   cdef numpy.ndarray[numpy.double_t, ndim=1] output = \
       numpy.PyArray_SimpleNewFromData(1, &dim, numpy.NPY_DOUBLE, GetMemory(X))
 
-  # Transfer memory ownership.
-  SetMemState[arma.Col[double]](X, 1)
-  PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
+  # Transfer memory ownership, if needed.
+  if GetMemState[arma.Col[double]](X) == 0:
+    SetMemState[arma.Col[double]](X, 1)
+    PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
 
   return output
 
@@ -239,8 +262,9 @@ cdef numpy.ndarray[numpy.npy_intp, ndim=1] col_to_numpy_s(arma.Col[size_t]& X) \
   cdef numpy.ndarray[numpy.npy_intp, ndim=1] output = \
       numpy.PyArray_SimpleNewFromData(1, &dim, numpy.NPY_INTP, GetMemory(X))
 
-  # Transfer memory ownership.
-  SetMemState[arma.Col[size_t]](X, 1)
-  PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
+  # Transfer memory ownership, if needed.
+  if GetMemState[arma.Col[size_t]](X) == 0:
+    SetMemState[arma.Col[size_t]](X, 1)
+    PyArray_ENABLEFLAGS(output, numpy.NPY_OWNDATA)
 
   return output

src/mlpack/bindings/python/mlpack/arma_util.hpp

@@ -24,6 +24,20 @@ void SetMemState(T& t, int state)
   const_cast<arma::uhword&>(t.mem_state) = state;
 }
 
+/**
+ * Get the memory state of the given Armadillo object.
+ */
+template<typename T>
+size_t GetMemState(T& t)
+{
+  // Fake the memory state if we are using preallocated memory---since we will
+  // end up copying that memory, NumPy can own it.
+  if (t.mem && t.n_elem <= arma::arma_config::mat_prealloc)
+    return 0;
+
+  return (size_t) t.mem_state;
+}
+
 /**
  * Return the matrix's allocated memory pointer, unless the matrix is using its
  * internal preallocated memory, in which case we copy that and return a

src/mlpack/bindings/python/mlpack/cli.pxd

@@ -20,6 +20,9 @@ cdef extern from "<mlpack/core/util/cli.hpp>" namespace "mlpack" nogil:
     @staticmethod
     (T&) GetParam[T](string) nogil except +
 
+    @staticmethod
+    bool HasParam(string) nogil except +
+
     @staticmethod
     void SetPassed(string) nogil except +
 
@@ -37,16 +40,13 @@ cdef extern from "<mlpack/core/util/cli.hpp>" namespace "mlpack" nogil:
 
 cdef extern from "<mlpack/bindings/python/mlpack/cli_util.hpp>" \
     namespace "mlpack::util" nogil:
-  void SetParam[T](string, const T&) nogil except +
-  void SetParamWithInfo[T](string, const T&, const bool*) nogil except +
+  void SetParam[T](string, T&) nogil except +
+  void SetParamPtr[T](string, T*, bool) nogil except +
+  void SetParamWithInfo[T](string, T&, const bool*) nogil except +
+  (T*) GetParamPtr[T](string) nogil except +
   (T&) GetParamWithInfo[T](string) nogil except +
   void EnableVerbose() nogil except +
   void DisableVerbose() nogil except +
   void DisableBacktrace() nogil except +
   void ResetTimers() nogil except +
   void EnableTimers() nogil except +
-
-cdef extern from "<mlpack/bindings/python/mlpack/move.hpp>" \
-    namespace "mlpack::util" nogil:
-  void MoveFromPtr[T](T&, T*) nogil except +
-  void MoveToPtr[T](T*, T&) nogil except +

src/mlpack/bindings/python/mlpack/cli_util.hpp

@@ -29,9 +29,27 @@ namespace util {
  * @param value Value to set parameter to.
  */
 template<typename T>
-inline void SetParam(const std::string& identifier, const T& value)
+inline void SetParam(const std::string& identifier, T& value)
 {
-  CLI::GetParam<T>(identifier) = value;
+  CLI::GetParam<T>(identifier) = std::move(value);
+}
+
+/**
+ * Set the parameter to the given value, given that the type is a pointer.
+ *
+ * This function exists to work around both Cython's lack of support for lvalue
+ * references and also its seeming lack of support for template pointer types.
+ *
+ * @param identifier Name of parameter.
+ * @param value Value to set parameter to.
+ * @param copy Whether or not the object should be copied.
+ */
+template<typename T>
+inline void SetParamPtr(const std::string& identifier,
+                        T* value,
+                        const bool copy)
+{
+  CLI::GetParam<T*>(identifier) = copy ? new T(*value) : value;
 }
 
 /**
@@ -39,19 +57,20 @@ inline void SetParam(const std::string& identifier, const T& value)
  */
 template<typename T>
 inline void SetParamWithInfo(const std::string& identifier,
-                             const T& matrix,
+                             T& matrix,
                              const bool* dims)
 {
   typedef typename std::tuple<data::DatasetInfo, T> TupleType;
   typedef typename T::elem_type eT;
 
   // The true type of the parameter is std::tuple<T, DatasetInfo>.
-  std::get<1>(CLI::GetParam<TupleType>(identifier)) = matrix;
+  const size_t dimensions = matrix.n_rows;
+  std::get<1>(CLI::GetParam<TupleType>(identifier)) = std::move(matrix);
   data::DatasetInfo& di = std::get<0>(CLI::GetParam<TupleType>(identifier));
-  di = data::DatasetInfo(matrix.n_rows);
+  di = data::DatasetInfo(dimensions);
 
   bool hasCategoricals = false;
-  for (size_t i = 0; i < matrix.n_rows; ++i)
+  for (size_t i = 0; i < dimensions; ++i)
   {
     if (dims[i])
     {
@@ -63,9 +82,10 @@ inline void SetParamWithInfo(const std::string& identifier,
   // Do we need to find how many categories we have?
   if (hasCategoricals)
   {
-    arma::vec maxs = arma::max(matrix, 1);
+    arma::vec maxs = arma::max(
+        std::get<1>(CLI::GetParam<TupleType>(identifier)), 1);
 
-    for (size_t i = 0; i < matrix.n_rows; ++i)
+    for (size_t i = 0; i < dimensions; ++i)
     {
       if (dims[i])
       {
@@ -81,6 +101,16 @@ inline void SetParamWithInfo(const std::string& identifier,
   }
 }
 
+/**
+ * Return a pointer.  This function exists to work around Cython's seeming lack
+ * of support for template pointer types.
+ */
+template<typename T>
+T* GetParamPtr(const std::string& paramName)
+{
+  return CLI::GetParam<T*>(paramName);
+}
+
 /**
  * Return the matrix part of a matrix + dataset info parameter.
  */

src/mlpack/bindings/python/mlpack/matrix_utils.py

@@ -37,7 +37,7 @@ try:
 except:
   buffer = memoryview
 
-def to_matrix(x, dtype=np.double):
+def to_matrix(x, dtype=np.double, copy=False):
   """
   Given some array-like X, return a numpy ndarray of the same type.
   """
@@ -48,11 +48,14 @@ def to_matrix(x, dtype=np.double):
     raise TypeError("given argument is not array-like")
 
   if (isinstance(x, np.ndarray) and x.dtype == dtype and x.flags.c_contiguous):
-    return x
+    if copy: # Copy the matrix if required.
+      return x.copy("C"), True
+    else:
+      return x, False
   else:
-    return np.array(x, copy=True, dtype=dtype, order='C')
+    return np.array(x, copy=True, dtype=dtype, order='C'), True
 
-def to_matrix_with_info(x, dtype):
+def to_matrix_with_info(x, dtype, copy=False):
   """
   Given some array-like X (which should be either a numpy ndarray or a pandas
   DataFrame, convert into a numpy matrix of the given dtype.
@@ -66,7 +69,12 @@ def to_matrix_with_info(x, dtype):
   if isinstance(x, np.ndarray):
     # It is already an ndarray, so the vector of info is all 0s (all numeric).
     d = np.zeros([x.shape[1]], dtype=np.bool)
-    return (x, d)
+
+    # Copy the matrix if needed.
+    if copy:
+      return (x.copy(order="C"), True, d)
+    else:
+      return (x, False, d)
 
   if isinstance(x, pd.DataFrame) or isinstance(x, pd.Series):
     # It's a pandas dataframe.  So we need to see if any of the dtypes are
@@ -79,8 +87,9 @@ def to_matrix_with_info(x, dtype):
        not np.dtype(str) in dtype_array and \
        not np.dtype(unicode) in dtype_array:
         # We can just return the matrix as-is; it's all numeric.
+        t = to_matrix(x, dtype=dtype, copy=copy)
         d = np.zeros([x.shape[1]], dtype=np.bool)
-        return (to_matrix(x), d)
+        return (t[0], t[1], d)
 
     if np.dtype(str) in dtype_array or np.dtype(unicode) in dtype_array:
       raise TypeError('cannot convert matrices with string types')
@@ -115,7 +124,10 @@ def to_matrix_with_info(x, dtype):
       catColumnIndices = [y.columns.get_loc(i) for i in catColumns]
       d[catColumnIndices] = 1
 
-    return (to_matrix(y.apply(pd.to_numeric)), d)
+    # We'll have to force the second part of the tuple (whether or not to take
+    # ownership) to true.
+    t = to_matrix(y.apply(pd.to_numeric), dtype=dtype)
+    return (t[0], True, d)
 
   if isinstance(x, list):
     # Get the number of dimensions.
@@ -126,7 +138,18 @@ def to_matrix_with_info(x, dtype):
       dims = len(x)
 
     d = np.zeros([dims])
-    return (np.array(x, dtype=dtype), d)
+    out = np.array(x, dtype=dtype, copy=copy) # Try to avoid copy...
+
+    # Since we don't have a great way to check if these are using the same
+    # memory location, we will probe manually (ugh).
+    oldval = x[0]
+    x[0] *= 2
+    alias = False
+    if out[0] == x[0]:
+      alias = True
+      x[0] = oldval
+
+    return (out, not alias, d)
 
   # If we got here, the type is not known.
   raise TypeError("given matrix is not a numpy ndarray or pandas DataFrame or "\

src/mlpack/bindings/python/mlpack/move.hpp

@@ -1,30 +0,0 @@
-/**
- * @file move.hpp
- * @author Ryan Curtin
- *
- * Utility function for Cython to use std::move.
- */
-#ifndef MLPACK_BINDINGS_PYTHON_CYTHON_MOVE_HPP
-#define MLPACK_BINDINGS_PYTHON_CYTHON_MOVE_HPP
-
-#include <utility>
-
-namespace mlpack {
-namespace util {
-
-template<typename T>
-void MoveToPtr(T* dest, T& src)
-{
-  *(dest) = std::move(src);
-}
-
-template<typename T>
-void MoveFromPtr(T& dest, T* src)
-{
-  dest = std::move(*src);
-}
-
-} // namespace util
-} // namespace mlpack
-
-#endif

src/mlpack/bindings/python/print_class_defn.hpp

@@ -108,7 +108,7 @@ void PrintClassDefn(const util::ParamData& d,
                     const void* /* input */,
                     void* /* output */)
 {
-  PrintClassDefn<T>(d);
+  PrintClassDefn<typename std::remove_pointer<T>::type>(d);
 }
 
 } // namespace python

src/mlpack/bindings/python/print_doc.hpp

@@ -39,7 +39,27 @@ void PrintDoc(const util::ParamData& d,
     oss << d.name << "_ (";
   else
     oss << d.name << " (";
-  oss << GetPythonType<T>(d) << "): " << d.desc;
+  oss << GetPythonType<typename std::remove_pointer<T>::type>(d) << "): "
+      << d.desc;
+
+  // Print a default, if possible.
+  if (!d.required)
+  {
+    if (d.cppType == "std::string")
+    {
+      oss << "  Default value '" << boost::any_cast<std::string>(d.value)
+          << "'.";
+    }
+    else if (d.cppType == "double")
+    {
+      oss << "  Default value " << boost::any_cast<double>(d.value) << ".";
+    }
+    else if (d.cppType == "int")
+    {
+      oss << "  Default value " << boost::any_cast<int>(d.value) << ".";
+    }
+  }
+
   std::cout << util::HyphenateString(oss.str(), indent + 4);
 }
 

src/mlpack/bindings/python/print_input_processing.hpp

@@ -31,6 +31,11 @@ void PrintInputProcessing(
     const typename boost::disable_if<std::is_same<T,
         std::tuple<data::DatasetInfo, arma::mat>>>::type* = 0)
 {
+  // The copy_all_inputs parameter must be handled first, and therefore is
+  // outside the scope of this code.
+  if (d.name == "copy_all_inputs")
+    return;
+
   const std::string prefix(indent, ' ');
 
   std::string def = "None";
@@ -104,7 +109,9 @@ void PrintInputProcessing(
    *
    * # Detect if the parameter was passed; set if so.
    * if param_name is not None:
-   *   param_name_mat = arma_numpy.numpy_to_mat_d(param_name)
+   *   param_name_tuple = to_matrix(param_name)
+   *   param_name_mat = arma_numpy.numpy_to_mat_d(param_name_tuple[0],
+   *       param_name_tuple[1])
    *   SetParam[mat](<const string> 'param_name', dereference(param_name_mat))
    *   CLI.SetPassed(<const string> 'param_name')
    */
@@ -114,27 +121,33 @@ void PrintInputProcessing(
   {
     std::cout << prefix << "if " << d.name << " is not None:" << std::endl;
 
+    std::cout << prefix << "  " << d.name << "_tuple = to_matrix(" << d.name
+        << ", dtype=" << GetNumpyType<typename T::elem_type>() << ", "
+        << "copy=CLI.HasParam('copy_all_inputs'))" << std::endl;
     std::cout << prefix << "  " << d.name << "_mat = arma_numpy.numpy_to_"
-        << GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(to_matrix("
-        << d.name << ", " << "dtype=" << GetNumpyType<typename T::elem_type>()
-        << "))" << std::endl;
+        << GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
+        << "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
     std::cout << prefix << "  SetParam[" << GetCythonType<T>(d) << "](<const "
         << "string> '" << d.name << "', dereference(" << d.name << "_mat))"
         << std::endl;
     std::cout << prefix << "  CLI.SetPassed(<const string> '" << d.name << "')"
         << std::endl;
+    std::cout << prefix << "  del " << d.name << "_mat";
   }
   else
   {
+    std::cout << prefix << d.name << "_tuple = to_matrix(" << d.name
+        << ", dtype=" << GetNumpyType<typename T::elem_type>() << ", "
+        << "copy=CLI.HasParam('copy_all_inputs'))" << std::endl;
     std::cout << prefix << d.name << "_mat = arma_numpy.numpy_to_"
-        << GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(to_matrix("
-        << d.name << ", " << "dtype=" << GetNumpyType<typename T::elem_type>()
-        << "))" << std::endl;
+        << GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
+        << "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
     std::cout << prefix << "SetParam[" << GetCythonType<T>(d) << "](<const "
         << "string> '" << d.name << "', dereference(" << d.name << "_mat))"
         << std::endl;
     std::cout << prefix << "CLI.SetPassed(<const string> '" << d.name << "')"
         << std::endl;
+    std::cout << prefix << "del " << d.name << "_mat";
   }
   std::cout << std::endl;
 }
@@ -161,12 +174,12 @@ void PrintInputProcessing(
    * # Detect if the parameter was passed; set if so.
    * if param_name is not None:
    *   try:
-   *     MoveFromPtr[Model](CLI.GetParam[Model]('param_name'),
-   *         (<ModelType?> param_name).modelptr)
+   *     SetParamPtr[Model]('param_name', (<ModelType?> param_name).modelptr,
+   *         CLI.HasParam('copy_all_inputs'))
    *   except TypeError as e:
    *     if type(param_name).__name__ == "ModelType":
-   *       MoveFromPtr[Model](CLI.GetParam[Model]('param_name'),
-   *           (<ModelType> param_name).modelptr)
+   *       SetParamPtr[Model]('param_name', (<ModelType> param_name).modelptr,
+   *           CLI.HasParam('copy_all_inputs'))
    *     else:
    *       raise e
    *   CLI.SetPassed(<const string> 'param_name')
@@ -177,15 +190,15 @@ void PrintInputProcessing(
   {
     std::cout << prefix << "if " << d.name << " is not None:" << std::endl;
     std::cout << prefix << "  try:" << std::endl;
-    std::cout << prefix << "    MoveFromPtr[" << strippedType
-        << "](CLI.GetParam[" << strippedType << "]('" << d.name << "'), (<"
-        << strippedType << "Type?> " << d.name << ").modelptr)" << std::endl;
+    std::cout << prefix << "    SetParamPtr[" << strippedType << "]('" << d.name
+        << "', (<" << strippedType << "Type?> " << d.name << ").modelptr, "
+        << "CLI.HasParam('copy_all_inputs'))" << std::endl;
     std::cout << prefix << "  except TypeError as e:" << std::endl;
     std::cout << prefix << "    if type(" << d.name << ").__name__ == '"
         << strippedType << "Type':" << std::endl;
-    std::cout << prefix << "      MoveFromPtr[" << strippedType
-        << "](CLI.GetParam[" << strippedType << "]('" << d.name << "'), (<"
-        << strippedType << "Type> " << d.name << ").modelptr)" << std::endl;
+    std::cout << prefix << "      SetParamPtr[" << strippedType << "]('"
+        << d.name << "', (<" << strippedType << "Type> " << d.name
+        << ").modelptr, CLI.HasParam('copy_all_inputs'))" << std::endl;
     std::cout << prefix << "    else:" << std::endl;
     std::cout << prefix << "      raise e" << std::endl;
     std::cout << prefix << "  CLI.SetPassed(<const string> '" << d.name << "')"
@@ -194,15 +207,15 @@ void PrintInputProcessing(
   else
   {
     std::cout << prefix << "try:" << std::endl;
-    std::cout << prefix << "  MoveFromPtr[" << strippedType << "](CLI.GetParam["
-        << strippedType << "]('" << d.name << "'), (<" << strippedType
-        << "Type?> " << d.name << ").modelptr)" << std::endl;
+    std::cout << prefix << "  SetParamPtr[" << strippedType << "]('" << d.name
+        << "', (<" << strippedType << "Type?> " << d.name << ").modelptr, "
+        << "CLI.HasParam('copy_all_inputs'))" << std::endl;
     std::cout << prefix << "except TypeError as e:" << std::endl;
     std::cout << prefix << "  if type(" << d.name << ").__name__ == '"
         << strippedType << "Type':" << std::endl;
-    std::cout << prefix << "    MoveFromPtr[" << strippedType
-        << "](CLI.GetParam[" << strippedType << "]('" << d.name << "'), (<"
-        << strippedType << "Type> " << d.name << ").modelptr)" << std::endl;
+    std::cout << prefix << "    SetParamPtr[" << strippedType << "]('" << d.name
+        << "', (<" << strippedType << "Type> " << d.name << ").modelptr, "
+        << "CLI.HasParam('copy_all_inputs'))" << std::endl;
     std::cout << prefix << "  else:" << std::endl;
     std::cout << prefix << "    raise e" << std::endl;
     std::cout << prefix << "CLI.SetPassed(<const string> '" << d.name << "')"
@@ -240,30 +253,34 @@ void PrintInputProcessing(
   {
     std::cout << prefix << "if " << d.name << " is not None:" << std::endl;
     std::cout << prefix << "  " << d.name << "_tuple = to_matrix_with_info("
-        << d.name << ", dtype=np.double)" << std::endl;
+        << d.name << ", dtype=np.double, copy=CLI.HasParam('copy_all_inputs'))"
+        << std::endl;
     std::cout << prefix << "  " << d.name << "_mat = arma_numpy.numpy_to_mat_d("
-        << d.name << "_tuple[0])" << std::endl;
-    std::cout << prefix << "  " << d.name << "_dims = " << d.name << "_tuple[1]"
+        << d.name << "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
+    std::cout << prefix << "  " << d.name << "_dims = " << d.name << "_tuple[2]"
         << std::endl;
     std::cout << prefix << "  SetParamWithInfo[arma.Mat[double]](<const string>"
         << " '" << d.name << "', dereference(" << d.name << "_mat), <const "
         << "bool*> " << d.name << "_dims.data)" << std::endl;
     std::cout << prefix << "  CLI.SetPassed(<const string> '" << d.name << "')"
         << std::endl;
+    std::cout << prefix << "  del " << d.name << "_mat" << std::endl;
   }
   else
   {
     std::cout << prefix << d.name << "_tuple = to_matrix_with_info(" << d.name
-        << ", dtype=np.double)" << std::endl;
+        << ", dtype=np.double, copy=CLI.HasParam('copy_all_inputs'))"
+        << std::endl;
     std::cout << prefix << d.name << "_mat = arma_numpy.numpy_to_mat_d("
-        << d.name << "_tuple[0])" << std::endl;
-    std::cout << prefix << d.name << "_dims = " << d.name << "_tuple[1]"
+        << d.name << "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
+    std::cout << prefix << d.name << "_dims = " << d.name << "_tuple[2]"
         << std::endl;
     std::cout << prefix << "SetParamWithInfo[arma.Mat[double]](<const string>"
         << " '" << d.name << "', dereference(" << d.name << "_mat), <const "
         << "bool*> " << d.name << "_dims.data)" << std::endl;
     std::cout << prefix << "CLI.SetPassed(<const string> '" << d.name << "')"
         << std::endl;
+    std::cout << prefix << "del " << d.name << "_mat" << std::endl;
   }
   std::cout << std::endl;
 }
@@ -284,7 +301,8 @@ void PrintInputProcessing(const util::ParamData& d,
                           const void* input,
                           void* /* output */)
 {
-  PrintInputProcessing<T>(d, *((size_t*) input));
+  PrintInputProcessing<typename std::remove_pointer<T>::type>(d,
+      *((size_t*) input));
 }
 
 } // namespace python

src/mlpack/bindings/python/print_output_processing.hpp

@@ -180,13 +180,47 @@ void PrintOutputProcessing(
      * This gives us code like:
      *
      * result = ModelType()
-     * MoveToPtr[Model]((<ModelType?> model).modelptr),
-     *     CLI.GetParam[Model]('name'))
+     * (<ModelType?> result).modelptr = GetParamPtr[Model]('name')
      */
     std::cout << prefix << "result = " << strippedType << "Type()" << std::endl;
-    std::cout << prefix << "MoveToPtr[" << strippedType << "]((<"
-        << strippedType << "Type?> result).modelptr, CLI.GetParam["
-        << strippedType << "]('" << d.name << "'))" << std::endl;
+    std::cout << prefix << "(<" << strippedType << "Type?> result).modelptr = "
+        << "GetParamPtr[" << strippedType << "]('" << d.name << "')"
+        << std::endl;
+
+    /**
+     * But we also have to check to ensure there aren't any input model
+     * parameters of the same type that could have the same model pointer.
+     * So we need to loop through all input parameters that have the same type,
+     * and double-check.
+     */
+    std::map<std::string, util::ParamData>& parameters = CLI::Parameters();
+    for (auto it = parameters.begin(); it != parameters.end(); ++it)
+    {
+      // Is it an input parameter of the same type?
+      const util::ParamData& data = it->second;
+      if (data.input && data.cppType == d.cppType && data.required)
+      {
+        std::cout << prefix << "if (<" << strippedType
+            << "Type> result).modelptr" << d.name << " == (<" << strippedType
+            << "Type> " << data.name << ").modelptr:" << std::endl;
+        std::cout << prefix << "  (<" << strippedType
+            << "Type> result).modelptr = <" << strippedType << "*> 0"
+            << std::endl;
+        std::cout << prefix << "  result = " << data.name << std::endl;
+      }
+      else if (data.input && data.cppType == d.cppType)
+      {
+        std::cout << prefix << "if " << data.name << " is not None:"
+            << std::endl;
+        std::cout << prefix << "  if (<" << strippedType
+            << "Type> result).modelptr" << d.name << " == (<" << strippedType
+            << "Type> " << data.name << ").modelptr:" << std::endl;
+        std::cout << prefix << "    (<" << strippedType
+            << "Type> result).modelptr = <" << strippedType << "*> 0"
+            << std::endl;
+        std::cout << prefix << "    result = " << data.name << std::endl;
+      }
+    }
   }
   else
   {
@@ -194,15 +228,50 @@ void PrintOutputProcessing(
      * This gives us code like:
      *
      * result['name'] = ModelType()
-     * MoveToPtr[Model*]((<ModelType?> result['name']).modelptr),
-     *     CLI.GetParam[Model]('name'))
+     * (<ModelType?> result['name']).modelptr = GetParamPtr[Model]('name'))
      */
     std::cout << prefix << "result['" << d.name << "'] = " << strippedType
         << "Type()" << std::endl;
-    std::cout << prefix << "MoveToPtr[" << strippedType << "]((<"
-        << strippedType << "Type?>" << " result['" << d.name
-        << "']).modelptr, CLI.GetParam[" << strippedType << "]('"
-        << d.name << "'))" << std::endl;
+    std::cout << prefix << "(<" << strippedType << "Type?> result['" << d.name
+        << "']).modelptr = GetParamPtr[" << strippedType << "]('" << d.name
+        << "')" << std::endl;
+
+    /**
+     * But we also have to check to ensure there aren't any input model
+     * parameters of the same type that could have the same model pointer.
+     * So we need to loop through all input parameters that have the same type,
+     * and double-check.
+     */
+    std::map<std::string, util::ParamData>& parameters = CLI::Parameters();
+    for (auto it = parameters.begin(); it != parameters.end(); ++it)
+    {
+      // Is it an input parameter of the same type?
+      const util::ParamData& data = it->second;
+      if (data.input && data.cppType == d.cppType && data.required)
+      {
+        std::cout << prefix << "if (<" << strippedType << "Type> result['"
+            << d.name << "']).modelptr == (<" << strippedType << "Type> "
+            << data.name << ").modelptr:" << std::endl;
+        std::cout << prefix << "  (<" << strippedType << "Type> result['"
+            << d.name << "']).modelptr = <" << strippedType << "*> 0"
+            << std::endl;
+        std::cout << prefix << "  result['" << d.name << "'] = " << data.name
+            << std::endl;
+      }
+      else if (data.input && data.cppType == d.cppType)
+      {
+        std::cout << prefix << "if " << data.name << " is not None:"
+            << std::endl;
+        std::cout << prefix << "  if (<" << strippedType << "Type> result['"
+            << d.name << "']).modelptr == (<" << strippedType << "Type> "
+            << data.name << ").modelptr:" << std::endl;
+        std::cout << prefix << "    (<" << strippedType << "Type> result['"
+            << d.name << "']).modelptr = <" << strippedType << "*> 0"
+            << std::endl;
+        std::cout << prefix << "    result['" << d.name << "'] = " << data.name
+            << std::endl;
+      }
+    }
   }
 }
 
@@ -227,7 +296,8 @@ void PrintOutputProcessing(const util::ParamData& d,
 {
   std::tuple<size_t, bool>* tuple = (std::tuple<size_t, bool>*) input;
 
-  PrintOutputProcessing<T>(d, std::get<0>(*tuple), std::get<1>(*tuple));
+  PrintOutputProcessing<typename std::remove_pointer<T>::type>(d,
+      std::get<0>(*tuple), std::get<1>(*tuple));
 }
 
 } // namespace python

src/mlpack/bindings/python/print_pyx.cpp

@@ -71,10 +71,10 @@ void PrintPYX(const ProgramDoc& programInfo,
   cout << "cimport arma" << endl;
   cout << "cimport arma_numpy" << endl;
   cout << "from cli cimport CLI" << endl;
-  cout << "from cli cimport SetParam, SetParamWithInfo" << endl;
+  cout << "from cli cimport SetParam, SetParamPtr, SetParamWithInfo, "
+      << "GetParamPtr" << endl;
   cout << "from cli cimport EnableVerbose, DisableVerbose, DisableBacktrace, "
       << "ResetTimers, EnableTimers" << endl;
-  cout << "from cli cimport MoveFromPtr, MoveToPtr" << endl;
   cout << "from matrix_utils import to_matrix, to_matrix_with_info" << endl;
   cout << "from serialization cimport SerializeIn, SerializeOut" << endl;
   cout << endl;
@@ -177,7 +177,14 @@ void PrintPYX(const ProgramDoc& programInfo,
   cout << "  DisableVerbose()" << endl;
 
   // Restore the parameters.
-  cout << "  CLI.RestoreSettings(\"" << programInfo.programName << "\")";
+  cout << "  CLI.RestoreSettings(\"" << programInfo.programName << "\")"
+      << endl;
+
+  // Determine whether or not we need to copy parameters.
+  cout << "  if copy_all_inputs:" << endl;
+  cout << "    SetParam[bool](<const string> 'copy_all_inputs', "
+      << "copy_all_inputs)" << endl;
+  cout << "    CLI.SetPassed(<const string> 'copy_all_inputs')" << endl;
 
   // Do any input processing.
   for (size_t i = 0; i < inputOptions.size(); ++i)

src/mlpack/bindings/python/py_option.hpp

@@ -58,8 +58,8 @@ class PyOption
     data.required = required;
     data.input = input;
     data.loaded = false;
-    // Only "verbose" will be persistent.
-    if (identifier == "verbose")
+    // Only "verbose" and "copy_all_inputs" will be persistent.
+    if (identifier == "verbose" || identifier == "copy_all_inputs")
       data.persistent = true;
     else
       data.persistent = false;

src/mlpack/bindings/python/tests/dataset_info_test.py

@@ -23,7 +23,7 @@ class TestToMatrix(unittest.TestCase):
     """
     d = pd.DataFrame(np.random.randn(100, 4), columns=list('abcd'))
 
-    m = to_matrix(d)
+    m, _ = to_matrix(d)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.shape[0], 100)
@@ -40,7 +40,7 @@ class TestToMatrix(unittest.TestCase):
     """
     d = pd.DataFrame({'a': range(5)})
 
-    m = to_matrix(d)
+    m, _ = to_matrix(d)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.shape[0], 5)
@@ -58,7 +58,7 @@ class TestToMatrix(unittest.TestCase):
     self.assertEqual(d['a'].dtype, int)
     self.assertEqual(d['b'].dtype, np.dtype(np.double))
 
-    m = to_matrix(d)
+    m, _ = to_matrix(d)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.dtype, np.dtype(np.double))
@@ -79,7 +79,7 @@ class TestToMatrix(unittest.TestCase):
          [0.07, 0.08, 0.09],
          [0.10, 0.11, 0.12]]
 
-    m = to_matrix(a)
+    m, _ = to_matrix(a)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.dtype, np.dtype(np.double))
@@ -100,7 +100,7 @@ class TestToMatrix(unittest.TestCase):
          [0.07, 0.08, 9],
          [0.10, 0.11, 12]]
 
-    m = to_matrix(a)
+    m, _ = to_matrix(a)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.dtype, np.dtype(np.double))
@@ -116,7 +116,7 @@ class TestToMatrix(unittest.TestCase):
     Make sure we can convert a numpy matrix without copying anything.
     """
     m1 = np.random.randn(100, 5)
-    m2 = to_matrix(m1)
+    m2, _ = to_matrix(m1)
 
     self.assertTrue(isinstance(m2, np.ndarray))
     self.assertEqual(m2.dtype, np.dtype(np.double))
@@ -144,7 +144,7 @@ class TestToMatrixWithInfo(unittest.TestCase):
     """
     d = pd.DataFrame(np.random.randn(100, 4), columns=list('abcd'))
 
-    m, dims = to_matrix_with_info(d, np.double)
+    m, _, dims = to_matrix_with_info(d, np.double)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.shape[0], 100)
@@ -167,7 +167,7 @@ class TestToMatrixWithInfo(unittest.TestCase):
     """
     d = pd.DataFrame({'a': range(5)})
 
-    m, dims = to_matrix_with_info(d, np.double)
+    m, _, dims = to_matrix_with_info(d, np.double)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.shape[0], 5)
@@ -188,7 +188,7 @@ class TestToMatrixWithInfo(unittest.TestCase):
     self.assertEqual(d['a'].dtype, int)
     self.assertEqual(d['b'].dtype, np.dtype(np.double))
 
-    m, dims = to_matrix_with_info(d, np.double)
+    m, _, dims = to_matrix_with_info(d, np.double)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.dtype, np.dtype(np.double))
@@ -213,7 +213,7 @@ class TestToMatrixWithInfo(unittest.TestCase):
          [0.07, 0.08, 0.09],
          [0.10, 0.11, 0.12]]
 
-    m, dims = to_matrix_with_info(a, np.double)
+    m, _, dims = to_matrix_with_info(a, np.double)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.dtype, np.dtype(np.double))
@@ -239,7 +239,7 @@ class TestToMatrixWithInfo(unittest.TestCase):
          [0.07, 0.08, 9],
          [0.10, 0.11, 12]]
 
-    m, dims = to_matrix_with_info(a, np.double)
+    m, _, dims = to_matrix_with_info(a, np.double)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.dtype, np.dtype(np.double))
@@ -260,7 +260,7 @@ class TestToMatrixWithInfo(unittest.TestCase):
     Make sure we can convert a numpy matrix without copying anything.
     """
     m1 = np.random.randn(100, 5)
-    m2, dims = to_matrix_with_info(m1, np.double)
+    m2, _, dims = to_matrix_with_info(m1, np.double)
 
     self.assertTrue(isinstance(m2, np.ndarray))
     self.assertEqual(m2.dtype, np.dtype(np.double))
@@ -284,7 +284,7 @@ class TestToMatrixWithInfo(unittest.TestCase):
     d = pd.DataFrame({"A": ["a", "b", "c", "a"] })
     d["A"] = d["A"].astype('category') # Convert to categorical.
 
-    m, dims = to_matrix_with_info(d, np.double)
+    m, _, dims = to_matrix_with_info(d, np.double)
 
     self.assertTrue(isinstance(m, np.ndarray))
     self.assertEqual(m.dtype, np.dtype(np.double))

src/mlpack/bindings/python/tests/test_python_binding.py

@@ -7,6 +7,7 @@ Test that passing types to Python bindings works successfully.
 import unittest
 import pandas as pd
 import numpy as np
+import copy
 
 from mlpack.test_python_binding import test_python_binding
 
@@ -94,11 +95,35 @@ class TestPythonBinding(unittest.TestCase):
     and the fifth forgotten.
     """
     x = np.random.rand(100, 5);
+    z = copy.copy(x)
 
     output = test_python_binding(string_in='hello',
                                  int_in=12,
                                  double_in=4.0,
-                                 matrix_in=x)
+                                 matrix_in=z)
+
+    self.assertEqual(output['matrix_out'].shape[0], 100)
+    self.assertEqual(output['matrix_out'].shape[1], 4)
+    self.assertEqual(output['matrix_out'].dtype, np.double)
+    for i in [0, 1, 3]:
+      for j in range(100):
+        self.assertEqual(x[j, i], output['matrix_out'][j, i])
+
+    for j in range(100):
+      self.assertEqual(2 * x[j, 2], output['matrix_out'][j, 2])
+
+  def testNumpyMatrixForceCopy(self):
+    """
+    The matrix we pass in, we should get back with the third dimension doubled
+    and the fifth forgotten.
+    """
+    x = np.random.rand(100, 5);
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 matrix_in=x,
+                                 copy_all_inputs=True)
 
     self.assertEqual(output['matrix_out'].shape[0], 100)
     self.assertEqual(output['matrix_out'].shape[1], 4)
@@ -139,16 +164,71 @@ class TestPythonBinding(unittest.TestCase):
     self.assertEqual(output['matrix_out'][2, 2], 26)
     self.assertEqual(output['matrix_out'][2, 3], 14)
 
+  def testArraylikeMatrixForceCopy(self):
+    """
+    Test that we can pass an arraylike matrix.
+    """
+    x = [[1, 2, 3, 4, 5],
+         [6, 7, 8, 9, 10],
+         [11, 12, 13, 14, 15]]
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 matrix_in=x,
+                                 copy_all_inputs=True)
+
+    self.assertEqual(output['matrix_out'].shape[0], 3)
+    self.assertEqual(output['matrix_out'].shape[1], 4)
+    self.assertEqual(len(x), 3)
+    self.assertEqual(len(x[0]), 5)
+    self.assertEqual(output['matrix_out'].dtype, np.double)
+    self.assertEqual(output['matrix_out'][0, 0], 1)
+    self.assertEqual(output['matrix_out'][0, 1], 2)
+    self.assertEqual(output['matrix_out'][0, 2], 6)
+    self.assertEqual(output['matrix_out'][0, 3], 4)
+    self.assertEqual(output['matrix_out'][1, 0], 6)
+    self.assertEqual(output['matrix_out'][1, 1], 7)
+    self.assertEqual(output['matrix_out'][1, 2], 16)
+    self.assertEqual(output['matrix_out'][1, 3], 9)
+    self.assertEqual(output['matrix_out'][2, 0], 11)
+    self.assertEqual(output['matrix_out'][2, 1], 12)
+    self.assertEqual(output['matrix_out'][2, 2], 26)
+    self.assertEqual(output['matrix_out'][2, 3], 14)
+
   def testNumpyUmatrix(self):
     """
     Same as testNumpyMatrix() but with an unsigned matrix.
     """
     x = np.random.randint(0, high=500, size=[100, 5])
+    z = copy.copy(x)
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 umatrix_in=z)
+
+    self.assertEqual(output['umatrix_out'].shape[0], 100)
+    self.assertEqual(output['umatrix_out'].shape[1], 4)
+    self.assertEqual(output['umatrix_out'].dtype, np.long)
+    for i in [0, 1, 3]:
+      for j in range(100):
+        self.assertEqual(x[j, i], output['umatrix_out'][j, i])
+
+    for j in range(100):
+      self.assertEqual(2 * x[j, 2], output['umatrix_out'][j, 2])
+
+  def testNumpyUmatrixForceCopy(self):
+    """
+    Same as testNumpyMatrix() but with an unsigned matrix.
+    """
+    x = np.random.randint(0, high=500, size=[100, 5])
 
     output = test_python_binding(string_in='hello',
                                  int_in=12,
                                  double_in=4.0,
-                                 umatrix_in=x)
+                                 umatrix_in=x,
+                                 copy_all_inputs=True)
 
     self.assertEqual(output['umatrix_out'].shape[0], 100)
     self.assertEqual(output['umatrix_out'].shape[1], 4)
@@ -189,16 +269,67 @@ class TestPythonBinding(unittest.TestCase):
     self.assertEqual(output['umatrix_out'][2, 2], 26)
     self.assertEqual(output['umatrix_out'][2, 3], 14)
 
+  def testArraylikeUmatrixForceCopy(self):
+    """
+    Test that we can pass an arraylike unsigned matrix.
+    """
+    x = [[1, 2, 3, 4, 5],
+         [6, 7, 8, 9, 10],
+         [11, 12, 13, 14, 15]]
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 umatrix_in=x,
+                                 copy_all_inputs=True)
+
+    self.assertEqual(output['umatrix_out'].shape[0], 3)
+    self.assertEqual(output['umatrix_out'].shape[1], 4)
+    self.assertEqual(len(x), 3)
+    self.assertEqual(len(x[0]), 5)
+    self.assertEqual(output['umatrix_out'].dtype, np.long)
+    self.assertEqual(output['umatrix_out'][0, 0], 1)
+    self.assertEqual(output['umatrix_out'][0, 1], 2)
+    self.assertEqual(output['umatrix_out'][0, 2], 6)
+    self.assertEqual(output['umatrix_out'][0, 3], 4)
+    self.assertEqual(output['umatrix_out'][1, 0], 6)
+    self.assertEqual(output['umatrix_out'][1, 1], 7)
+    self.assertEqual(output['umatrix_out'][1, 2], 16)
+    self.assertEqual(output['umatrix_out'][1, 3], 9)
+    self.assertEqual(output['umatrix_out'][2, 0], 11)
+    self.assertEqual(output['umatrix_out'][2, 1], 12)
+    self.assertEqual(output['umatrix_out'][2, 2], 26)
+    self.assertEqual(output['umatrix_out'][2, 3], 14)
+
   def testCol(self):
     """
     Test a column vector input parameter.
     """
     x = np.random.rand(100)
+    z = copy.copy(x)
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 col_in=z)
+
+    self.assertEqual(output['col_out'].shape[0], 100)
+    self.assertEqual(output['col_out'].dtype, np.double)
+
+    for i in range(100):
+      self.assertEqual(output['col_out'][i], x[i] * 2)
+
+  def testColForceCopy(self):
+    """
+    Test a column vector input parameter.
+    """
+    x = np.random.rand(100)
 
     output = test_python_binding(string_in='hello',
                                  int_in=12,
                                  double_in=4.0,
-                                 col_in=x)
+                                 col_in=x,
+                                 copy_all_inputs=True)
 
     self.assertEqual(output['col_out'].shape[0], 100)
     self.assertEqual(output['col_out'].dtype, np.double)
@@ -211,11 +342,29 @@ class TestPythonBinding(unittest.TestCase):
     Test an unsigned column vector input parameter.
     """
     x = np.random.randint(0, high=500, size=100)
+    z = copy.copy(x)
 
     output = test_python_binding(string_in='hello',
                                  int_in=12,
                                  double_in=4.0,
-                                 ucol_in=x)
+                                 ucol_in=z)
+
+    self.assertEqual(output['ucol_out'].shape[0], 100)
+    self.assertEqual(output['ucol_out'].dtype, np.long)
+    for i in range(100):
+      self.assertEqual(output['ucol_out'][i], x[i] * 2)
+
+  def testUcolForceCopy(self):
+    """
+    Test an unsigned column vector input parameter.
+    """
+    x = np.random.randint(0, high=500, size=100)
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 ucol_in=x,
+                                 copy_all_inputs=True)
 
     self.assertEqual(output['ucol_out'].shape[0], 100)
     self.assertEqual(output['ucol_out'].dtype, np.long)
@@ -227,11 +376,30 @@ class TestPythonBinding(unittest.TestCase):
     Test a row vector input parameter.
     """
     x = np.random.rand(100)
+    z = copy.copy(x)
 
     output = test_python_binding(string_in='hello',
                                  int_in=12,
                                  double_in=4.0,
-                                 row_in=x)
+                                 row_in=z)
+
+    self.assertEqual(output['row_out'].shape[0], 100)
+    self.assertEqual(output['row_out'].dtype, np.double)
+
+    for i in range(100):
+      self.assertEqual(output['row_out'][i], x[i] * 2)
+
+  def testRowForceCopy(self):
+    """
+    Test a row vector input parameter.
+    """
+    x = np.random.rand(100)
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 row_in=x,
+                                 copy_all_inputs=True)
 
     self.assertEqual(output['row_out'].shape[0], 100)
     self.assertEqual(output['row_out'].dtype, np.double)
@@ -244,11 +412,30 @@ class TestPythonBinding(unittest.TestCase):
     Test an unsigned row vector input parameter.
     """
     x = np.random.randint(0, high=500, size=100)
+    z = copy.copy(x)
 
     output = test_python_binding(string_in='hello',
                                  int_in=12,
                                  double_in=4.0,
-                                 urow_in=x)
+                                 urow_in=z)
+
+    self.assertEqual(output['urow_out'].shape[0], 100)
+    self.assertEqual(output['urow_out'].dtype, np.long)
+
+    for i in range(100):
+      self.assertEqual(output['urow_out'][i], x[i] * 2)
+
+  def testUrowForceCopy(self):
+    """
+    Test an unsigned row vector input parameter.
+    """
+    x = np.random.randint(0, high=500, size=100)
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 urow_in=x,
+                                 copy_all_inputs=True)
 
     self.assertEqual(output['urow_out'].shape[0], 100)
     self.assertEqual(output['urow_out'].dtype, np.long)
@@ -261,11 +448,31 @@ class TestPythonBinding(unittest.TestCase):
     Test that we can pass a matrix with all numeric features.
     """
     x = np.random.rand(100, 10)
+    z = copy.copy(x)
 
     output = test_python_binding(string_in='hello',
                                  int_in=12,
                                  double_in=4.0,
-                                 matrix_and_info_in=x)
+                                 matrix_and_info_in=z)
+
+    self.assertEqual(output['matrix_and_info_out'].shape[0], 100)
+    self.assertEqual(output['matrix_and_info_out'].shape[1], 10)
+
+    for i in range(10):
+      for j in range(100):
+        self.assertEqual(output['matrix_and_info_out'][j, i], x[j, i] * 2.0)
+
+  def testMatrixAndInfoNumpyForceCopy(self):
+    """
+    Test that we can pass a matrix with all numeric features.
+    """
+    x = np.random.rand(100, 10)
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 matrix_and_info_in=x,
+                                 copy_all_inputs=True)
 
     self.assertEqual(output['matrix_and_info_out'].shape[0], 100)
     self.assertEqual(output['matrix_and_info_out'].shape[1], 10)
@@ -281,11 +488,38 @@ class TestPythonBinding(unittest.TestCase):
     x = pd.DataFrame(np.random.rand(10, 4), columns=list('abcd'))
     x['e'] = pd.Series(['a', 'b', 'c', 'd', 'a', 'b', 'e', 'c', 'a', 'b'],
         dtype='category')
+    z = copy.copy(x)
 
     output = test_python_binding(string_in='hello',
                                  int_in=12,
                                  double_in=4.0,
-                                 matrix_and_info_in=x)
+                                 matrix_and_info_in=z)
+
+    self.assertEqual(output['matrix_and_info_out'].shape[0], 10)
+    self.assertEqual(output['matrix_and_info_out'].shape[1], 5)
+
+    cols = list('abcde')
+
+    for i in range(4):
+      for j in range(10):
+        self.assertEqual(output['matrix_and_info_out'][j, i], z[cols[i]][j] * 2)
+
+    for j in range(10):
+      self.assertEqual(output['matrix_and_info_out'][j, 4], z[cols[4]][j])
+
+  def testMatrixAndInfoPandasForceCopy(self):
+    """
+    Test that we can pass a matrix with some categorical features.
+    """
+    x = pd.DataFrame(np.random.rand(10, 4), columns=list('abcd'))
+    x['e'] = pd.Series(['a', 'b', 'c', 'd', 'a', 'b', 'e', 'c', 'a', 'b'],
+        dtype='category')
+
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 matrix_and_info_in=x,
+                                 copy_all_inputs=True)
 
     self.assertEqual(output['matrix_and_info_out'].shape[0], 10)
     self.assertEqual(output['matrix_and_info_out'].shape[1], 5)
@@ -345,5 +579,29 @@ class TestPythonBinding(unittest.TestCase):
 
     self.assertEqual(output2['model_bw_out'], 20.0)
 
+  def testModelForceCopy(self):
+    """
+    First create a GaussianKernel object, then send it back and make sure we get
+    the right double value.
+    """
+    output = test_python_binding(string_in='hello',
+                                 int_in=12,
+                                 double_in=4.0,
+                                 build_model=True)
+
+    output2 = test_python_binding(string_in='hello',
+                                  int_in=12,
+                                  double_in=4.0,
+                                  model_in=output['model_out'],
+                                  copy_all_inputs=True)
+
+    output3 = test_python_binding(string_in='hello',
+                                  int_in=12,
+                                  double_in=4.0,
+                                  model_in=output['model_out'])
+
+    self.assertEqual(output2['model_bw_out'], 20.0)
+    self.assertEqual(output3['model_bw_out'], 20.0)
+
 if __name__ == '__main__':
   unittest.main()

src/mlpack/bindings/python/tests/test_python_binding_main.cpp

@@ -172,13 +172,13 @@ static void mlpackMain()
   // If we got a request to build a model, then build it.
   if (CLI::HasParam("build_model"))
   {
-    CLI::GetParam<GaussianKernel>("model_out") = GaussianKernel(10.0);
+    CLI::GetParam<GaussianKernel*>("model_out") = new GaussianKernel(10.0);
   }
 
   // If we got an input model, double the bandwidth and output that.
   if (CLI::HasParam("model_in"))
   {
     CLI::GetParam<double>("model_bw_out") =
-        CLI::GetParam<GaussianKernel>("model_in").Bandwidth() * 2.0;
+        CLI::GetParam<GaussianKernel*>("model_in")->Bandwidth() * 2.0;
   }
 }

src/mlpack/bindings/tests/CMakeLists.txt

@@ -1,8 +1,12 @@
 # Define the files we need to compile.
 # Anything not in this list will not be compiled into mlpack.
 set(SOURCES
+  clean_memory.hpp
+  clean_memory.cpp
   test_option.hpp
   ignore_check.hpp
+  delete_allocated_memory.hpp
+  get_allocated_memory.hpp
   get_param.hpp
   get_printable_param.hpp
   get_printable_param_impl.hpp

src/mlpack/bindings/tests/clean_memory.cpp

@@ -0,0 +1,54 @@
+/**
+ * @file clean_memory.cpp
+ * @author Ryan Curtin
+ *
+ * Delete any pointers held by the CLI object.
+ */
+#include "clean_memory.hpp"
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace bindings {
+namespace tests {
+
+/**
+ * Delete any pointers held by the CLI object.
+ */
+void CleanMemory()
+{
+  // If we are holding any pointers, then we "own" them.  But we may hold the
+  // same pointer twice, so we have to be careful to not delete it multiple
+  // times.
+  std::unordered_map<void*, const util::ParamData*> memoryAddresses;
+  auto it = CLI::Parameters().begin();
+  while (it != CLI::Parameters().end())
+  {
+    const util::ParamData& data = it->second;
+
+    void* result;
+    CLI::GetSingleton().functionMap[data.tname]["GetAllocatedMemory"](data,
+        NULL, (void*) &result);
+    if (result != NULL && memoryAddresses.count(result) == 0)
+      memoryAddresses[result] = &data;
+
+    ++it;
+  }
+
+  // Now we have all the unique addresses that need to be deleted.
+  std::unordered_map<void*, const util::ParamData*>::const_iterator it2;
+  it2 = memoryAddresses.begin();
+  while (it2 != memoryAddresses.end())
+  {
+    const util::ParamData& data = *(it2->second);
+
+    CLI::GetSingleton().functionMap[data.tname]["DeleteAllocatedMemory"](data,
+        NULL, NULL);
+
+    ++it2;
+  }
+}
+
+} // namespace tests
+} // namespace bindings
+} // namespace mlpack

src/mlpack/bindings/tests/clean_memory.hpp

@@ -0,0 +1,24 @@
+/**
+ * @file clean_memory.hpp
+ * @author Ryan Curtin
+ *
+ * Delete any unique pointers that are held by the CLI object.  This is similar
+ * to the code in end_program.hpp.
+ */
+#ifndef MLPACK_BINDINGS_TESTS_CLEAN_MEMORY_HPP
+#define MLPACK_BINDINGS_TESTS_CLEAN_MEMORY_HPP
+
+namespace mlpack {
+namespace bindings {
+namespace tests {
+
+/**
+ * Delete any unique pointers that are held by the CLI object.
+ */
+void CleanMemory();
+
+} // namespace tests
+} // namespace bindings
+} // namespace mlpack
+
+#endif

src/mlpack/bindings/tests/delete_allocated_memory.hpp

@@ -0,0 +1,56 @@
+/**
+ * @file delete_allocated_memory.hpp
+ * @author Ryan Curtin
+ *
+ * If any memory has been allocated by the parameter, delete it.
+ */
+#ifndef MLPACK_BINDINGS_CLI_DELETE_ALLOCATED_MEMORY_HPP
+#define MLPACK_BINDINGS_CLI_DELETE_ALLOCATED_MEMORY_HPP
+
+#include <mlpack/core/util/param_data.hpp>
+
+namespace mlpack {
+namespace bindings {
+namespace tests {
+
+template<typename T>
+void DeleteAllocatedMemoryImpl(
+    const util::ParamData& /* d */,
+    const typename boost::disable_if<data::HasSerialize<T>>::type* = 0,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0)
+{
+  // Do nothing.
+}
+
+template<typename T>
+void DeleteAllocatedMemoryImpl(
+    const util::ParamData& /* d */,
+    const typename boost::enable_if<arma::is_arma_type<T>>::type* = 0)
+{
+  // Do nothing.
+}
+
+template<typename T>
+void DeleteAllocatedMemoryImpl(
+    const util::ParamData& d,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
+{
+  // Delete the allocated memory (hopefully we actually own it).
+  delete *boost::any_cast<T*>(&d.value);
+}
+
+template<typename T>
+void DeleteAllocatedMemory(
+    const util::ParamData& d,
+    const void* /* input */,
+    void* /* output */)
+{
+  DeleteAllocatedMemoryImpl<typename std::remove_pointer<T>::type>(d);
+}
+
+} // namespace tests
+} // namespace bindings
+} // namespace mlpack
+
+#endif

src/mlpack/bindings/tests/get_allocated_memory.hpp

@@ -0,0 +1,57 @@
+/**
+ * @file get_allocated_memory.hpp
+ * @author Ryan Curtin
+ *
+ * If the parameter has a type that may need to be deleted, return the address
+ * of that object.  Otherwise return NULL.
+ */
+#ifndef MLPACK_BINDINGS_CLI_GET_ALLOCATED_MEMORY_HPP
+#define MLPACK_BINDINGS_CLI_GET_ALLOCATED_MEMORY_HPP
+
+#include <mlpack/core/util/param_data.hpp>
+
+namespace mlpack {
+namespace bindings {
+namespace tests {
+
+template<typename T>
+void* GetAllocatedMemory(
+    const util::ParamData& /* d */,
+    const typename boost::disable_if<data::HasSerialize<T>>::type* = 0,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0)
+{
+  return NULL;
+}
+
+template<typename T>
+void* GetAllocatedMemory(
+    const util::ParamData& /* d */,
+    const typename boost::enable_if<arma::is_arma_type<T>>::type* = 0)
+{
+  return NULL;
+}
+
+template<typename T>
+void* GetAllocatedMemory(
+    const util::ParamData& d,
+    const typename boost::disable_if<arma::is_arma_type<T>>::type* = 0,
+    const typename boost::enable_if<data::HasSerialize<T>>::type* = 0)
+{
+  // Here we have a model; return its memory location.
+  return *boost::any_cast<T*>(&d.value);
+}
+
+template<typename T>
+void GetAllocatedMemory(const util::ParamData& d,
+                        const void* /* input */,
+                        void* output)
+{
+  *((void**) output) =
+      GetAllocatedMemory<typename std::remove_pointer<T>::type>(d);
+}
+
+} // namespace tests
+} // namespace bindings
+} // namespace mlpack
+
+#endif

src/mlpack/bindings/tests/get_printable_param.hpp

@@ -72,7 +72,8 @@ void GetPrintableParam(const util::ParamData& data,
                        const void* /* input */,
                        void* output)
 {
-  *((std::string*) output) = GetPrintableParam<T>(data);
+  *((std::string*) output) =
+      GetPrintableParam<typename std::remove_pointer<T>::type>(data);
 }
 
 } // namespace tests

src/mlpack/bindings/tests/test_option.hpp

@@ -18,6 +18,8 @@
 #include <mlpack/core/util/cli.hpp>
 #include "get_printable_param.hpp"
 #include "get_param.hpp"
+#include "get_allocated_memory.hpp"
+#include "delete_allocated_memory.hpp"
 
 namespace mlpack {
 namespace bindings {
@@ -90,6 +92,10 @@ class TestOption
     CLI::GetSingleton().functionMap[tname]["GetPrintableParam"] =
         &GetPrintableParam<N>;
     CLI::GetSingleton().functionMap[tname]["GetParam"] = &GetParam<N>;
+    CLI::GetSingleton().functionMap[tname]["GetAllocatedMemory"] =
+        &GetAllocatedMemory<N>;
+    CLI::GetSingleton().functionMap[tname]["DeleteAllocatedMemory"] =
+        &DeleteAllocatedMemory<N>;
 
     CLI::Add(std::move(data));
 

src/mlpack/core.hpp

@@ -230,6 +230,7 @@
  *   - Manish Kumar <manish887kr@gmail.com>
  *   - Haritha Sreedharan Nair <haritha1313@gmail.com>
  *   - Sourabh Varshney <sourabhvarshney111@gmail.com>
+ *   - Nikhil Goel <nikhilgoel199797@gmail.com>
  */
 
 // First, include all of the prerequisites.

src/mlpack/core/data/CMakeLists.txt

@@ -6,6 +6,7 @@ set(SOURCES
   extension.hpp
   format.hpp
   has_serialize.hpp
+  is_naninf.hpp
   load_csv.hpp
   load_csv.cpp
   load.hpp

src/mlpack/core/data/is_naninf.hpp

@@ -0,0 +1,67 @@
+/**
+ * @file is_naninf.hpp
+ * @author Ryan Curtin
+ *
+ * This is an adapted version of Conrad Sanderson's implementation of
+ * arma::diskio::convert_naninf() from Armadillo.  It is here so as to avoid
+ * using Armadillo internal functionality.
+ */
+#ifndef MLPACK_CORE_DATA_HAS_NANINF_HPP
+#define MLPACK_CORE_DATA_HAS_NANINF_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace data {
+
+/**
+ * See if the token is a NaN or an Inf, and if so, set the value accordingly and
+ * return a boolean representing whether or not it is.
+ */
+template<typename T>
+inline bool IsNaNInf(T& val, const std::string& token)
+{
+  // See if the token represents a NaN or Inf.
+  if ((token.length() == 3) || (token.length() == 4))
+  {
+    const bool neg = (token[0] == '-');
+    const bool pos = (token[0] == '+');
+
+    const size_t offset = ((neg || pos) && (token.length() == 4)) ? 1 : 0;
+
+    const std::string token2 = token.substr(offset, 3);
+
+    if ((token2 == "inf") || (token2 == "Inf") || (token2 == "INF"))
+    {
+      if (std::numeric_limits<T>::has_infinity)
+      {
+        val = (!neg) ? std::numeric_limits<T>::infinity() :
+            -std::numeric_limits<T>::infinity();
+      }
+      else
+      {
+        val = (!neg) ? std::numeric_limits<T>::max() :
+            -std::numeric_limits<T>::max();
+      }
+
+      return true;
+    }
+    else if ((token2 == "nan") || (token2 == "Nan") || (token2 == "NaN") ||
+        (token2 == "NAN") )
+    {
+      if (std::numeric_limits<T>::has_quiet_NaN)
+        val = std::numeric_limits<T>::quiet_NaN();
+      else
+        val = T(0);
+
+      return true;
+    }
+  }
+
+  return false;
+}
+
+} // namespace data
+} // namespace mlpack
+
+#endif

src/mlpack/core/data/load_arff_impl.hpp

@@ -16,6 +16,7 @@
 #include "load_arff.hpp"
 
 #include <boost/algorithm/string/trim.hpp>
+#include "is_naninf.hpp"
 
 namespace mlpack {
 namespace data {
@@ -197,7 +198,7 @@ void LoadARFF(const std::string& filename,
         if (token.fail())
         {
           // Check for NaN or inf.
-          if (!arma::diskio::convert_naninf(val, token.str()))
+          if (!IsNaNInf(val, token.str()))
           {
             // Okay, it's not NaN or inf.  If it's '?', we issue a specific
             // error, otherwise we issue a general error.

src/mlpack/core/optimizers/CMakeLists.txt

@@ -11,6 +11,7 @@ set(DIRS
   iqn
   lbfgs
   line_search
+  problems
   proximal
   parallel_sgd
   rmsprop

src/mlpack/core/optimizers/fw/constr_lpball.hpp

@@ -115,7 +115,7 @@ class ConstrLpBallSolver
       else
         s = arma::abs(v);
 
-      arma::uword k;
+      arma::uword k = 0;
       s.max(k);  // k is the linear index of the largest element.
       s.zeros();
       s(k) = - mlpack::math::Sign(v(k));

src/mlpack/core/optimizers/lbfgs/CMakeLists.txt

@@ -2,8 +2,6 @@ set(SOURCES
   lbfgs_impl.hpp
   lbfgs.hpp
   lbfgs.cpp
-  test_functions.hpp
-  test_functions.cpp
 )
 
 set(DIR_SRCS)

src/mlpack/core/optimizers/lbfgs/test_functions.cpp

@@ -1,284 +0,0 @@
-/**
- * @file test_functions.cpp
- * @author Ryan Curtin
- *
- * Implementations of the test functions defined in test_functions.hpp.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#include "test_functions.hpp"
-
-using namespace mlpack::optimization::test;
-
-//
-// RosenbrockFunction implementation
-//
-
-RosenbrockFunction::RosenbrockFunction()
-{
-  initialPoint.set_size(2, 1);
-  initialPoint[0] = -1.2;
-  initialPoint[1] = 1;
-}
-
-/**
- * Calculate the objective function.
- */
-double RosenbrockFunction::Evaluate(const arma::mat& coordinates)
-{
-  double x1 = coordinates[0];
-  double x2 = coordinates[1];
-
-  double objective = /* f1(x) */ 100 * std::pow(x2 - std::pow(x1, 2), 2) +
-                     /* f2(x) */ std::pow(1 - x1, 2);
-
-  return objective;
-}
-
-/**
- * Calculate the gradient.
- */
-void RosenbrockFunction::Gradient(const arma::mat& coordinates,
-                                  arma::mat& gradient)
-{
-  // f'_{x1}(x) = -2 (1 - x1) + 400 (x1^3 - (x2 x1))
-  // f'_{x2}(x) = 200 (x2 - x1^2)
-
-  double x1 = coordinates[0];
-  double x2 = coordinates[1];
-
-  gradient.set_size(2, 1);
-  gradient[0] = -2 * (1 - x1) + 400 * (std::pow(x1, 3) - x2 * x1);
-  gradient[1] = 200 * (x2 - std::pow(x1, 2));
-}
-
-const arma::mat& RosenbrockFunction::GetInitialPoint() const
-{
-  return initialPoint;
-}
-
-//
-// WoodFunction implementation
-//
-
-WoodFunction::WoodFunction()
-{
-  initialPoint.set_size(4, 1);
-  initialPoint[0] = -3;
-  initialPoint[1] = -1;
-  initialPoint[2] = -3;
-  initialPoint[3] = -1;
-}
-
-/**
- * Calculate the objective function.
- */
-double WoodFunction::Evaluate(const arma::mat& coordinates)
-{
-  // For convenience; we assume these temporaries will be optimized out.
-  double x1 = coordinates[0];
-  double x2 = coordinates[1];
-  double x3 = coordinates[2];
-  double x4 = coordinates[3];
-
-  double objective = /* f1(x) */ 100 * std::pow(x2 - std::pow(x1, 2), 2) +
-                     /* f2(x) */ std::pow(1 - x1, 2) +
-                     /* f3(x) */ 90 * std::pow(x4 - std::pow(x3, 2), 2) +
-                     /* f4(x) */ std::pow(1 - x3, 2) +
-                     /* f5(x) */ 10 * std::pow(x2 + x4 - 2, 2) +
-                     /* f6(x) */ (1.0 / 10.0) * std::pow(x2 - x4, 2);
-
-  return objective;
-}
-
-/**
- * Calculate the gradient.
- */
-void WoodFunction::Gradient(const arma::mat& coordinates,
-                            arma::mat& gradient)
-{
-  // For convenience; we assume these temporaries will be optimized out.
-  double x1 = coordinates[0];
-  double x2 = coordinates[1];
-  double x3 = coordinates[2];
-  double x4 = coordinates[3];
-
-  // f'_{x1}(x) = 400 (x1^3 - x2 x1) - 2 (1 - x1)
-  // f'_{x2}(x) = 200 (x2 - x1^2) + 20 (x2 + x4 - 2) + (1 / 5) (x2 - x4)
-  // f'_{x3}(x) = 360 (x3^3 - x4 x3) - 2 (1 - x3)
-  // f'_{x4}(x) = 180 (x4 - x3^2) + 20 (x2 + x4 - 2) - (1 / 5) (x2 - x4)
-  gradient.set_size(4, 1);
-  gradient[0] = 400 * (std::pow(x1, 3) - x2 * x1) - 2 * (1 - x1);
-  gradient[1] = 200 * (x2 - std::pow(x1, 2)) + 20 * (x2 + x4 - 2) +
-      (1.0 / 5.0) * (x2 - x4);
-  gradient[2] = 360 * (std::pow(x3, 3) - x4 * x3) - 2 * (1 - x3);
-  gradient[3] = 180 * (x4 - std::pow(x3, 2)) + 20 * (x2 + x4 - 2) -
-      (1.0 / 5.0) * (x2 - x4);
-}
-
-const arma::mat& WoodFunction::GetInitialPoint() const
-{
-  return initialPoint;
-}
-
-//
-// GeneralizedRosenbrockFunction implementation
-//
-
-GeneralizedRosenbrockFunction::GeneralizedRosenbrockFunction(int n) :
-    n(n),
-    visitationOrder(arma::linspace<arma::Row<size_t>>(0, n - 2, n - 1))
-{
-  initialPoint.set_size(n, 1);
-  for (int i = 0; i < n; i++) // Set to [-1.2 1 -1.2 1 ...].
-  {
-    if (i % 2 == 1)
-      initialPoint[i] = -1.2;
-    else
-      initialPoint[i] = 1;
-  }
-}
-
-/**
- * Shuffle the data points.
- */
-void GeneralizedRosenbrockFunction::Shuffle()
-{
-  visitationOrder = arma::shuffle(arma::linspace<arma::Row<size_t>>(0, n - 2,
-      n - 1));
-}
-
-/**
- * Calculate the objective function.
- */
-double GeneralizedRosenbrockFunction::Evaluate(const arma::mat& coordinates)
-    const
-{
-  double fval = 0;
-  for (int i = 0; i < (n - 1); i++)
-  {
-    fval += 100 * std::pow(std::pow(coordinates[i], 2) -
-        coordinates[i + 1], 2) + std::pow(1 - coordinates[i], 2);
-  }
-
-  return fval;
-}
-
-/**
- * Calculate the gradient.
- */
-void GeneralizedRosenbrockFunction::Gradient(const arma::mat& coordinates,
-                                             arma::mat& gradient) const
-{
-  gradient.set_size(n);
-  for (int i = 0; i < (n - 1); i++)
-  {
-    gradient[i] = 400 * (std::pow(coordinates[i], 3) - coordinates[i] *
-        coordinates[i + 1]) + 2 * (coordinates[i] - 1);
-
-    if (i > 0)
-      gradient[i] += 200 * (coordinates[i] - std::pow(coordinates[i - 1], 2));
-  }
-
-  gradient[n - 1] = 200 * (coordinates[n - 1] -
-      std::pow(coordinates[n - 2], 2));
-}
-
-//! Calculate the objective function of one of the individual functions.
-double GeneralizedRosenbrockFunction::Evaluate(const arma::mat& coordinates,
-                                               const size_t i,
-                                               const size_t batchSize) const
-{
-  double objective = 0.0;
-  for (size_t j = i; j < i + batchSize; ++j)
-  {
-    const size_t p = visitationOrder[j];
-    objective += 100 * std::pow((std::pow(coordinates[p], 2)
-        - coordinates[p + 1]), 2) + std::pow(1 - coordinates[p], 2);
-  }
-
-  return objective;
-}
-
-//! Calculate the gradient of one of the individual functions.
-void GeneralizedRosenbrockFunction::Gradient(const arma::mat& coordinates,
-                                             const size_t i,
-                                             arma::mat& gradient,
-                                             const size_t batchSize) const
-{
-  gradient.zeros(n);
-
-  for (size_t j = i; j < i + batchSize; ++j)
-  {
-    const size_t p = visitationOrder[j];
-    gradient[p] = 400 * (std::pow(coordinates[p], 3) - coordinates[p] *
-        coordinates[p + 1]) + 2 * (coordinates[p] - 1);
-    gradient[p + 1] = 200 * (coordinates[p + 1] - std::pow(coordinates[p], 2));
-  }
-}
-
-void GeneralizedRosenbrockFunction::Gradient(const arma::mat& coordinates,
-                                             const size_t i,
-                                             arma::sp_mat& gradient) const
-{
-  gradient.set_size(n);
-
-  const size_t p = visitationOrder[i];
-
-  gradient[p] = 400 * (std::pow(coordinates[p], 3) - coordinates[p] *
-      coordinates[p + 1]) + 2 * (coordinates[p] - 1);
-  gradient[p + 1] = 200 * (coordinates[p + 1] - std::pow(coordinates[p], 2));
-}
-
-const arma::mat& GeneralizedRosenbrockFunction::GetInitialPoint() const
-{
-  return initialPoint;
-}
-
-//
-// RosenbrockWoodFunction implementation
-//
-
-RosenbrockWoodFunction::RosenbrockWoodFunction() : rf(4), wf()
-{
-  initialPoint.set_size(4, 2);
-  initialPoint.col(0) = rf.GetInitialPoint();
-  initialPoint.col(1) = wf.GetInitialPoint();
-}
-
-/**
- * Calculate the objective function.
- */
-double RosenbrockWoodFunction::Evaluate(const arma::mat& coordinates)
-{
-  double objective = rf.Evaluate(coordinates.col(0)) +
-                     wf.Evaluate(coordinates.col(1));
-
-  return objective;
-}
-
-/***
- * Calculate the gradient.
- */
-void RosenbrockWoodFunction::Gradient(const arma::mat& coordinates,
-                                      arma::mat& gradient)
-{
-  gradient.set_size(4, 2);
-
-  arma::vec grf(4);
-  arma::vec gwf(4);
-
-  rf.Gradient(coordinates.col(0), grf);
-  wf.Gradient(coordinates.col(1), gwf);
-
-  gradient.col(0) = grf;
-  gradient.col(1) = gwf;
-}
-
-const arma::mat& RosenbrockWoodFunction::GetInitialPoint() const
-{
-  return initialPoint;
-}

src/mlpack/core/optimizers/lbfgs/test_functions.hpp

@@ -1,174 +0,0 @@
-/**
- * @file test_functions.hpp
- * @author Ryan Curtin
- *
- * A collection of functions to test optimizers (in this case, L-BFGS).  These
- * come from the following paper:
- *
- * "Testing Unconstrained Optimization Software"
- *  Jorge J. Moré, Burton S. Garbow, and Kenneth E. Hillstrom. 1981.
- *  ACM Trans. Math. Softw. 7, 1 (March 1981), 17-41.
- *  http://portal.acm.org/citation.cfm?id=355934.355936
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef MLPACK_CORE_OPTIMIZERS_LBFGS_TEST_FUNCTIONS_HPP
-#define MLPACK_CORE_OPTIMIZERS_LBFGS_TEST_FUNCTIONS_HPP
-
-#include <mlpack/prereqs.hpp>
-
-// To fulfill the template policy class 'FunctionType', we must implement
-// the following:
-//
-//   FunctionType(); // constructor
-//   void Gradient(const arma::mat& coordinates, arma::mat& gradient);
-//   double Evaluate(const arma::mat& coordinates);
-//   const arma::mat& GetInitialPoint();
-//
-// Note that we are using an arma::mat instead of the more intuitive and
-// expected arma::vec.  This is because L-BFGS will also optimize matrices.
-// However, remember that an arma::vec is simply an (n x 1) arma::mat.  You can
-// use either internally but the L-BFGS method requires arma::mat& to be passed
-// (C++ does not allow implicit reference casting to subclasses).
-
-namespace mlpack {
-namespace optimization {
-namespace test {
-
-/**
- * The Rosenbrock function, defined by
- *  f(x) = f1(x) + f2(x)
- *  f1(x) = 100 (x2 - x1^2)^2
- *  f2(x) = (1 - x1)^2
- *  x_0 = [-1.2, 1]
- *
- * This should optimize to f(x) = 0, at x = [1, 1].
- *
- * "An automatic method for finding the greatest or least value of a function."
- *   H.H. Rosenbrock.  1960.  Comput. J. 3., 175-184.
- */
-class RosenbrockFunction
-{
- public:
-  RosenbrockFunction(); // initialize initial point
-
-  double Evaluate(const arma::mat& coordinates);
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
-
-  const arma::mat& GetInitialPoint() const;
-
- private:
-  arma::mat initialPoint;
-};
-
-/**
- * The Wood function, defined by
- *  f(x) = f1(x) + f2(x) + f3(x) + f4(x) + f5(x) + f6(x)
- *  f1(x) = 100 (x2 - x1^2)^2
- *  f2(x) = (1 - x1)^2
- *  f3(x) = 90 (x4 - x3^2)^2
- *  f4(x) = (1 - x3)^2
- *  f5(x) = 10 (x2 + x4 - 2)^2
- *  f6(x) = (1 / 10) (x2 - x4)^2
- *  x_0 = [-3, -1, -3, -1]
- *
- * This should optimize to f(x) = 0, at x = [1, 1, 1, 1].
- *
- * "A comparative study of nonlinear programming codes."
- *   A.R. Colville.  1968.  Rep. 320-2949, IBM N.Y. Scientific Center.
- */
-class WoodFunction
-{
- public:
-  WoodFunction(); // initialize initial point
-
-  double Evaluate(const arma::mat& coordinates);
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
-
-  const arma::mat& GetInitialPoint() const;
-
- private:
-  arma::mat initialPoint;
-};
-
-/**
- * The Generalized Rosenbrock function in n dimensions, defined by
- *  f(x) = sum_i^{n - 1} (f(i)(x))
- *  f_i(x) = 100 * (x_i^2 - x_{i + 1})^2 + (1 - x_i)^2
- *  x_0 = [-1.2, 1, -1.2, 1, ...]
- *
- * This should optimize to f(x) = 0, at x = [1, 1, 1, 1, ...].
- *
- * This function can also be used for stochastic gradient descent (SGD) as a
- * decomposable function (DecomposableFunctionType), so there are other
- * overloads of Evaluate() and Gradient() implemented, as well as
- * NumFunctions().
- *
- * "An analysis of the behavior of a glass of genetic adaptive systems."
- *   K.A. De Jong.  Ph.D. thesis, University of Michigan, 1975.
- */
-class GeneralizedRosenbrockFunction
-{
- public:
-  /***
-   * Set the dimensionality of the extended Rosenbrock function.
-   *
-   * @param n Number of dimensions for the function.
-   */
-  GeneralizedRosenbrockFunction(int n);
-
-  void Shuffle();
-
-  double Evaluate(const arma::mat& coordinates) const;
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const;
-
-  size_t NumFunctions() const { return n - 1; }
-  double Evaluate(const arma::mat& coordinates,
-                  const size_t i,
-                  const size_t batchSize = 1) const;
-  void Gradient(const arma::mat& coordinates,
-                const size_t i,
-                arma::mat& gradient,
-                const size_t batchSize = 1) const;
-
-  void Gradient(const arma::mat& coordinates,
-                const size_t i,
-                arma::sp_mat& gradient) const;
-
-  const arma::mat& GetInitialPoint() const;
-
- private:
-  arma::mat initialPoint;
-  int n; // Dimensionality
-  arma::Row<size_t> visitationOrder; // For shuffling.
-};
-
-/**
- * The Generalized Rosenbrock function in 4 dimensions with the Wood Function in
- * four dimensions.  In this function we are actually optimizing a 2x4 matrix of
- * coordinates, not a vector.
- */
-class RosenbrockWoodFunction
-{
- public:
-  RosenbrockWoodFunction(); // initialize initial point
-
-  double Evaluate(const arma::mat& coordinates);
-  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
-
-  const arma::mat& GetInitialPoint() const;
-
- private:
-  arma::mat initialPoint;
-  GeneralizedRosenbrockFunction rf;
-  WoodFunction wf;
-};
-
-} // namespace test
-} // namespace optimization
-} // namespace mlpack
-
-#endif // MLPACK_CORE_OPTIMIZERS_LBFGS_TEST_FUNCTIONS_HPP

src/mlpack/core/optimizers/problems/CMakeLists.txt

@@ -0,0 +1,43 @@
+set(SOURCES
+  booth_function.hpp
+  booth_function.cpp
+  bukin_function.hpp
+  bukin_function.cpp
+  colville_function.hpp
+  colville_function.cpp
+  drop_wave_function.hpp
+  drop_wave_function.cpp
+  easom_function.hpp
+  easom_function.cpp
+  eggholder_function.hpp
+  eggholder_function.cpp
+  generalized_rosenbrock_function.hpp
+  generalized_rosenbrock_function.cpp
+  matyas_function.hpp
+  matyas_function.cpp
+  mc_cormick_function.hpp
+  mc_cormick_function.cpp
+  rastrigin_function.hpp
+  rastrigin_function.cpp
+  rosenbrock_function.hpp
+  rosenbrock_wood_function.hpp
+  rosenbrock_wood_function.cpp
+  rosenbrock_function.cpp
+  schwefel_function.hpp
+  schwefel_function.cpp
+  sgd_test_function.hpp
+  sgd_test_function.cpp
+  sphere_function.hpp
+  sphere_function.cpp
+  styblinski_tang_function.hpp
+  styblinski_tang_function.cpp
+  wood_function.hpp
+  wood_function.cpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/core/optimizers/problems/booth_function.cpp

@@ -0,0 +1,58 @@
+/**
+ * @file booth_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Booth function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "booth_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+BoothFunction::BoothFunction() { /* Nothing to do here */ }
+
+void BoothFunction::Shuffle() { /* Nothing to do here */ }
+
+double BoothFunction::Evaluate(const arma::mat& coordinates,
+                               const size_t /* begin */,
+                               const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  const double objective = std::pow(x1 + 2 * x2 - 7, 2) +
+      std::pow(2 * x1 + x2 - 5, 2);
+
+  return objective;
+}
+
+double BoothFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void BoothFunction::Gradient(const arma::mat& coordinates,
+                             const size_t /* begin */,
+                             arma::mat& gradient,
+                             const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  gradient.set_size(2, 1);
+  gradient(0) = 10 * x1 + 8 * x2 - 34;
+  gradient(1) = 8 * x1 + 10 * x2 - 38;
+}
+
+void BoothFunction::Gradient(const arma::mat& coordinates, arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, 1);
+}

src/mlpack/core/optimizers/problems/booth_function.hpp

@@ -0,0 +1,105 @@
+/**
+ * @file booth_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Booth function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_BOOTH_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_BOOTH_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Booth function, defined by
+ *
+ * \f[
+ * f(x) = (x_1 + 2x_2 - 7)^2 + (2x_1 + x_2 - 5)^2
+ * \f]
+ *
+ * This should optimize to f(x) = 0, at x = [1, 3].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class BoothFunction
+{
+ public:
+  //! Initialize the BoothFunction.
+  BoothFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-9; -9"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_BOOTH_FUNCTION_HPP

src/mlpack/core/optimizers/problems/bukin_function.cpp

@@ -0,0 +1,62 @@
+/**
+ * @file bukin_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Bukin function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "bukin_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+BukinFunction::BukinFunction(const double epsilon) : epsilon(epsilon)
+{ /* Nothing to do here */ }
+
+void BukinFunction::Shuffle() { /* Nothing to do here */ }
+
+double BukinFunction::Evaluate(const arma::mat& coordinates,
+                               const size_t /* begin */,
+                               const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  const double objective = 100 * std::sqrt(std::abs(x2 - 0.01 *
+      std::pow(x1, 2))) + 0.01 * std::abs(x1 + 10);
+
+  return objective;
+}
+
+double BukinFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void BukinFunction::Gradient(const arma::mat& coordinates,
+                             const size_t /* begin */,
+                             arma::mat& gradient,
+                             const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  gradient.set_size(2, 1);
+  gradient(0) = (0.01 * (x1 + 10.0)) / (std::abs(x1 + 10.0) + epsilon) -
+      (x1 * (x2 - 0.01 * std::pow(x1, 2))) / std::pow(std::abs(x2 - 0.01 *
+      std::pow(x1, 2)), 1.5);
+  gradient(1) = (50 * (x2 - 0.01 * std::pow(x1, 2))) /
+      std::pow(std::abs(x2 - 0.01 * std::pow(x1, 2)), 1.5);
+}
+
+void BukinFunction::Gradient(const arma::mat& coordinates, arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/bukin_function.hpp

@@ -0,0 +1,119 @@
+/**
+ * @file bukin_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Booth function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_BUKIN_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_BUKIN_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Bukin function, defined by
+ *
+ * \f[
+ * f(x) = 100 * \sqrt(\left|x_2 - 0.01 * x_1^2 \right|) +
+ *    0.01 * \left|x_1 + 10 \right|
+ * \f]
+ *
+ * This should optimize to f(x) = 0, at x = [-10, 1].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class BukinFunction
+{
+ public:
+  /*
+   * Initialize the BukinFunction.
+   *
+   * @param epsilon Coefficient to avoid division by zero (numerical stability).
+   */
+  BukinFunction(const double epsilon = 1e-8);
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-10; -2.0"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+
+  //! Get the value used for numerical stability.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used for numerical stability.
+  double& Epsilon() { return epsilon; }
+
+ private:
+  //! The value used for numerical stability.
+  double epsilon;
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_BUKIN_FUNCTION_HPP

src/mlpack/core/optimizers/problems/colville_function.cpp

@@ -0,0 +1,67 @@
+/**
+ * @file colville_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Coville function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "colville_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+ColvilleFunction::ColvilleFunction() { /* Nothing to do here */ }
+
+void ColvilleFunction::Shuffle() { /* Nothing to do here */ }
+
+double ColvilleFunction::Evaluate(const arma::mat& coordinates,
+                                  const size_t /* begin */,
+                                  const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+  const double x3 = coordinates(2);
+  const double x4 = coordinates(3);
+
+  const double objective = 100 * std::pow(std::pow(x1, 2) - x2, 2) +
+      std::pow(x1 - 1, 2) + std::pow(x3 - 1, 2) + 90 *
+      std::pow(std::pow(x3, 2) - x4, 2) + 10.1 * (std::pow(x2 - 1, 2) +
+      std::pow(x4 - 1, 2)) + 19.8 * (x2 - 1) * (x4 - 1);
+
+  return objective;
+}
+
+double ColvilleFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void ColvilleFunction::Gradient(const arma::mat& coordinates,
+                                const size_t /* begin */,
+                                arma::mat& gradient,
+                                const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+  const double x3 = coordinates(2);
+  const double x4 = coordinates(3);
+
+  gradient.set_size(4, 1);
+  gradient(0) = 2 * (200 * x1 * (std::pow(x1, 2) - x2) + x1 - 1);
+  gradient(1) = 19.8 * x4 - 200 * std::pow(x1, 2) + 220.2 * x2 - 40;
+  gradient(2) = 2 * (180 * x3 * (std::pow(x3, 2) - x4) + x3 - 1);
+  gradient(3) = 200.2 * x4 + 19.8 * x2 - 180 * std::pow(x3, 2) - 40;
+}
+
+void ColvilleFunction::Gradient(const arma::mat& coordinates,
+                                arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/colville_function.hpp

@@ -0,0 +1,106 @@
+/**
+ * @file colville_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Colville function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_COLVILLE_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_COLVILLE_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Colville function, defined by
+ *
+ * \f[
+ * f(x) = 100(x_1^2 - x_2)^2 + (x_1 - 1)^2 + (x_3 - 1)^2 + 90 * (x_3^2 - x_4)^2
+ *    + 10.1 * ((x_2-1)^2 + (x_4 - 1)^2) + 19.8 * (x_2 - 1) * (x_4 - 1)
+ * \f]
+ *
+ * This should optimize to f(x) = 0, at x = [1, 1, 1, 1].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class ColvilleFunction
+{
+ public:
+  //! Initialize the ColvilleFunction.
+  ColvilleFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-5; 3; 1; -9"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_COLVILLE_FUNCTION_HPP

src/mlpack/core/optimizers/problems/drop_wave_function.cpp

@@ -0,0 +1,71 @@
+/**
+ * @file drop_wave_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Drop-Wave function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "drop_wave_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+DropWaveFunction::DropWaveFunction() { /* Nothing to do here */ }
+
+void DropWaveFunction::Shuffle() { /* Nothing to do here */ }
+
+double DropWaveFunction::Evaluate(const arma::mat& coordinates,
+                                  const size_t /* begin */,
+                                  const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  const double objective = -1.0 * (1.0 + std::cos(12.0 *
+      std::sqrt(std::pow(x1, 2) + std::pow(x2, 2)))) /
+      (0.5 * (std::pow(x1, 2) + std::pow(x2, 2)) + 2.0);
+
+  return objective;
+}
+
+double DropWaveFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, 1);
+}
+
+void DropWaveFunction::Gradient(const arma::mat& coordinates,
+                                const size_t /* begin */,
+                                arma::mat& gradient,
+                                const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  gradient.set_size(2, 1);
+  gradient(0) = (12.0 * x1 * std::sin(12.0 * std::sqrt(std::pow(x1, 2) +
+      std::pow(x2, 2)))) / (std::sqrt(std::pow(x1, 2) + std::pow(x2, 2)) *
+      (0.5 * (std::pow(x1, 2) + std::pow(x2, 2)) + 2)) -
+      (x1 * (-1.0 * std::cos(12.0 * std::sqrt(std::pow(x1, 2) +
+      std::pow(x2, 2))) -1.0)) / std::pow(0.5 *
+      (std::pow(x1, 2) + std::pow(x2, 2)) + 2, 2);
+
+  gradient(1) = (12.0 * x2 * std::sin(12.0 * std::sqrt(std::pow(x1, 2) +
+      std::pow(x2, 2)))) / (std::sqrt(std::pow(x1, 2) + std::pow(x2, 2)) *
+      (0.5 * (std::pow(x1, 2) + std::pow(x2, 2)) + 2)) -
+      (x2 * (-1.0 * std::cos(12.0 * std::sqrt(std::pow(x1, 2) +
+      std::pow(x2, 2))) -1.0)) / std::pow(0.5 *
+      (std::pow(x1, 2) + std::pow(x2, 2)) + 2, 2);
+}
+
+void DropWaveFunction::Gradient(const arma::mat& coordinates,
+                                arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, 1);
+}

src/mlpack/core/optimizers/problems/drop_wave_function.hpp

@@ -0,0 +1,105 @@
+/**
+ * @file drop_wave_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Drop-Wave function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_DROP_WAVE_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_DROP_WAVE_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Drop-Wave function, defined by
+ *
+ * \f[
+ * f(x) = - (1 + \cos(12 * \sqrt(x_1^2 + x_2^2))) / (0.5 * (x_1^2 + x_2^2) + 2)
+ * \f]
+ *
+ * This should optimize to f(x) = 0, at x = [0, 0].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class DropWaveFunction
+{
+ public:
+  //! Initialize the DropWaveFunction.
+  DropWaveFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("0.5; 0.5"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_DROP_WAVE_FUNCTION_HPP

src/mlpack/core/optimizers/problems/easom_function.cpp

@@ -0,0 +1,67 @@
+/**
+ * @file easom_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Easom function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "easom_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+EasomFunction::EasomFunction() { /* Nothing to do here */ }
+
+void EasomFunction::Shuffle() { /* Nothing to do here */ }
+
+double EasomFunction::Evaluate(const arma::mat& coordinates,
+                               const size_t /* begin */,
+                               const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  const double objective = -std::cos(x1) * std::cos(x2) *
+      std::exp(-1.0 * std::pow(x1 - M_PI, 2) - std::pow(x2 - M_PI, 2));
+
+  return objective;
+}
+
+double EasomFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void EasomFunction::Gradient(const arma::mat& coordinates,
+                             const size_t /* begin */,
+                             arma::mat& gradient,
+                             const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  gradient.set_size(2, 1);
+  gradient(0) = 2 * (x1 - M_PI) *
+      std::exp(-1.0 * std::pow(x1 - M_PI, 2) - std::pow(x2 - M_PI, 2)) *
+      std::cos(x1) * std::cos(x2) +
+      std::exp(-1.0 * std::pow(x1 - M_PI, 2) - std::pow(x2 - M_PI, 2)) *
+      std::sin(x1) * std::cos(x2);
+
+  gradient(1) = 2 * (x2 - M_PI) *
+      std::exp(-1.0 * std::pow(x1 - M_PI, 2) - std::pow(x2 - M_PI, 2)) *
+      std::cos(x1) * std::cos(x2) +
+      std::exp(-1.0 * std::pow(x1 - M_PI, 2) - std::pow(x2 - M_PI, 2)) *
+      std::cos(x1) * std::sin(x2);
+}
+
+void EasomFunction::Gradient(const arma::mat& coordinates, arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/easom_function.hpp

@@ -0,0 +1,105 @@
+/**
+ * @file easom_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Booth function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_EASOM_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_EASOM_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Easom function, defined by
+ *
+ * \f[
+ * f(x) = -1.0 * \cos(x_1) * \cos(x_2) * \exp(-(x_1 - \pi)^2 - (x_2 - \pi)^2)
+ * \f]
+ *
+ * This should optimize to f(x) = -1, at x = [3.14, 3.14].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class EasomFunction
+{
+ public:
+  //! Initialize the EasomFunction.
+  EasomFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-90.0; 90.0"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_EASOM_FUNCTION_HPP

src/mlpack/core/optimizers/problems/eggholder_function.cpp

@@ -0,0 +1,71 @@
+/**
+ * @file eggholder_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Eggholder function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "eggholder_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+EggholderFunction::EggholderFunction() { /* Nothing to do here */ }
+
+void EggholderFunction::Shuffle() { /* Nothing to do here */ }
+
+double EggholderFunction::Evaluate(const arma::mat& coordinates,
+                                   const size_t /* begin */,
+                                   const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  const double objective = -1.0 * (x2 + 47) * std::sin(std::sqrt(
+      std::abs(x2 + x1 / 2 + 47))) - x1 * std::sin(std::sqrt(
+      std::abs(x1 - (x2 + 47))));
+
+  return objective;
+}
+
+double EggholderFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void EggholderFunction::Gradient(const arma::mat& coordinates,
+                                 const size_t /* begin */,
+                                 arma::mat& gradient,
+                                 const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  gradient.set_size(2, 1);
+  gradient(0) = -1.0 * std::sin(std::sqrt(std::abs(x1 - x2 - 47))) -
+      (x1 * (x1 - x2 - 47) * std::cos(std::sqrt(std::abs(x1 - x2 - 47)))) /
+      std::pow(2 * std::abs(x1 - x2 - 47), 1.5) -
+      ((x1 + 47) * (x1 / 2 + x2 + 47) *
+      std::cos(std::sqrt(std::abs(x1 / 2 + x2  + 47)))) /
+      (4 * std::pow(std::abs(x1 / 2 + x2  + 47), 1.5));
+
+  gradient(1) = -1.0 * std::sin(std::sqrt(std::abs(x1 / 2 + x2 + 47))) -
+      (x1 * (x1 - x2 - 47) * std::cos(std::sqrt(std::abs(x1 - x2 - 47)))) /
+      std::pow(2 * std::abs(x1 - x2 - 47), 1.5) -
+      ((x1 + 47) * (x1 / 2 + x2 + 47) *
+      std::cos(std::sqrt(std::abs(x1 / 2 + x2  + 47)))) /
+      (4 * std::pow(std::abs(x1 / 2 + x2  + 47), 1.5));
+}
+
+void EggholderFunction::Gradient(const arma::mat& coordinates,
+                                 arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/eggholder_function.hpp

@@ -0,0 +1,106 @@
+/**
+ * @file eggholder_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Eggholder function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_EGGHOLDER_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_EGGHOLDER_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Eggholder function, defined by
+ *
+ * \f[
+ * f(x) = -(x_2 + 47) * \sin(\sqrt(\left|x_2 + x_1 / 2 + 47\right|)) - x_1 *
+ *    \sin(\sqrt(\left|x_1-(x_2 + 47)\right|))
+ * \f]
+ *
+ * This should optimize to f(x) = -959.6407, at x = [512, 404.2319].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class EggholderFunction
+{
+ public:
+  //! Initialize the EggholderFunction.
+  EggholderFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-333; -333"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_EGGHOLDER_FUNCTION_HPP

src/mlpack/core/optimizers/problems/generalized_rosenbrock_function.cpp

@@ -0,0 +1,115 @@
+/**
+ * @file generalized_rosenbrock_function.cpp
+ * @author Ryan Curtin
+ * @author Marcus Edel
+ *
+ * Implementation of the Generalized-Rosenbrock function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "generalized_rosenbrock_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+GeneralizedRosenbrockFunction::GeneralizedRosenbrockFunction(const size_t n) :
+    n(n),
+    visitationOrder(arma::linspace<arma::Row<size_t> >(0, n - 1, n))
+
+{
+  initialPoint.set_size(n, 1);
+  for (size_t i = 0; i < n; i++) // Set to [-1.2 1 -1.2 1 ...].
+  {
+    if (i % 2 == 1)
+    {
+      initialPoint(i) = -1.2;
+    }
+    else
+    {
+      initialPoint(i) = 1;
+    }
+  }
+}
+
+void GeneralizedRosenbrockFunction::Shuffle()
+{
+  visitationOrder = arma::shuffle(arma::linspace<arma::Row<size_t>>(0, n - 2,
+      n - 1));
+}
+
+double GeneralizedRosenbrockFunction::Evaluate(const arma::mat& coordinates,
+                                               const size_t begin,
+                                               const size_t batchSize) const
+{
+  double objective = 0.0;
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    objective += 100 * std::pow((std::pow(coordinates[p], 2)
+        - coordinates[p + 1]), 2) + std::pow(1 - coordinates[p], 2);
+  }
+
+  return objective;
+}
+
+double GeneralizedRosenbrockFunction::Evaluate(const arma::mat& coordinates)
+    const
+{
+  double fval = 0;
+  for (size_t i = 0; i < (n - 1); i++)
+  {
+    fval += 100 * std::pow(std::pow(coordinates[i], 2) -
+        coordinates[i + 1], 2) + std::pow(1 - coordinates[i], 2);
+  }
+
+  return fval;
+}
+
+void GeneralizedRosenbrockFunction::Gradient(const arma::mat& coordinates,
+                                             const size_t begin,
+                                             arma::mat& gradient,
+                                             const size_t batchSize) const
+{
+  gradient.zeros(n);
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    gradient[p] = 400 * (std::pow(coordinates[p], 3) - coordinates[p] *
+        coordinates[p + 1]) + 2 * (coordinates[p] - 1);
+    gradient[p + 1] = 200 * (coordinates[p + 1] - std::pow(coordinates[p], 2));
+  }
+}
+
+void GeneralizedRosenbrockFunction::Gradient(const arma::mat& coordinates,
+                                             arma::mat& gradient) const
+{
+  gradient.set_size(n);
+  for (size_t i = 0; i < (n - 1); i++)
+  {
+    gradient[i] = 400 * (std::pow(coordinates[i], 3) - coordinates[i] *
+        coordinates[i + 1]) + 2 * (coordinates[i] - 1);
+
+    if (i > 0)
+      gradient[i] += 200 * (coordinates[i] - std::pow(coordinates[i - 1], 2));
+  }
+
+  gradient[n - 1] = 200 * (coordinates[n - 1] -
+      std::pow(coordinates[n - 2], 2));
+}
+
+void GeneralizedRosenbrockFunction::Gradient(const arma::mat& coordinates,
+                                             const size_t begin,
+                                             arma::sp_mat& gradient) const
+{
+  gradient.set_size(n);
+
+  const size_t p = visitationOrder[begin];
+
+  gradient[p] = 400 * (std::pow(coordinates[p], 3) - coordinates[p] *
+      coordinates[p + 1]) + 2 * (coordinates[p] - 1);
+  gradient[p + 1] = 200 * (coordinates[p + 1] - std::pow(coordinates[p], 2));
+}

src/mlpack/core/optimizers/problems/generalized_rosenbrock_function.hpp

@@ -0,0 +1,135 @@
+/**
+ * @file generalized_rosenbrock_function.hpp
+ * @author Ryan Curtin
+ * @author Marcus Edel
+ *
+ * Definition of the Generalized Rosenbrock function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_GENERALIZED_ROSENBROCK_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_GENERALIZED_ROSENBROCK_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Generalized Rosenbrock function in n dimensions, defined by
+ *  f(x) = sum_i^{n - 1} (f(i)(x))
+ *  f_i(x) = 100 * (x_i^2 - x_{i + 1})^2 + (1 - x_i)^2
+ *  x_0 = [-1.2, 1, -1.2, 1, ...]
+ *
+ * This should optimize to f(x) = 0, at x = [1, 1, 1, 1, ...].
+ *
+ * This function can also be used for stochastic gradient descent (SGD) as a
+ * decomposable function (DecomposableFunctionType), so there are other
+ * overloads of Evaluate() and Gradient() implemented, as well as
+ * NumFunctions().
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @phdthesis{Jong1975,
+ *   title  = {Analysis of the behavior of a class of genetic adaptive
+ *             systems},
+ *   author = {De Jong, Kenneth Alan},
+ *   school = {Queensland University of Technology},
+ *   year   = {1975},
+ *   type   = {{PhD} dissertation},
+ * }
+ * @endcode
+ */
+class GeneralizedRosenbrockFunction
+{
+ public:
+  /*
+   * Initialize the GeneralizedRosenbrockFunction.
+   *
+   * @param n Number of dimensions for the function.
+   */
+  GeneralizedRosenbrockFunction(const size_t n);
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return n - 1; }
+
+  //! Get the starting point.
+  const arma::mat& GetInitialPoint() const { return initialPoint;}
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize = 1) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize = 1) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::sp_mat& gradient) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const;
+
+ private:
+  //! Locally-stored Initial point.
+  arma::mat initialPoint;
+
+  //! //! Number of dimensions for the function.
+  size_t n;
+
+  //! For shuffling.
+  arma::Row<size_t> visitationOrder;
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_GENERALIZED_ROSENBROCK_FUNCTION_HPP

src/mlpack/core/optimizers/problems/matyas_function.cpp

@@ -0,0 +1,58 @@
+/**
+ * @file matyas_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Matyas function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "matyas_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+MatyasFunction::MatyasFunction() { /* Nothing to do here */ }
+
+void MatyasFunction::Shuffle() { /* Nothing to do here */ }
+
+double MatyasFunction::Evaluate(const arma::mat& coordinates,
+                                const size_t /* begin */,
+                                const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  const double objective = 0.26 * (pow(x1, 2) + std::pow(x2, 2)) -
+    0.48 * x1 * x2;
+
+  return objective;
+}
+
+double MatyasFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void MatyasFunction::Gradient(const arma::mat& coordinates,
+                              const size_t /* begin */,
+                              arma::mat& gradient,
+                              const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  gradient.set_size(2, 1);
+  gradient(0) = 0.52 * x1 - 48 * x2;
+  gradient(1) = 0.52 * x2 - 0.48 * x1;
+}
+
+void MatyasFunction::Gradient(const arma::mat& coordinates, arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/matyas_function.hpp

@@ -0,0 +1,105 @@
+/**
+ * @file matyas_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Matyas function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_MATYAS_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_MATYAS_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Matyas function, defined by
+ *
+ * \f[
+ * f(x) = 0.26 * (x_1^2 + x_2^2) - 0.48 * x_1 * x_2
+ * \f]
+ *
+ * This should optimize to f(x) = 0, at x = [0, 0].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class MatyasFunction
+{
+ public:
+  //! Initialize the MatyasFunction.
+  MatyasFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-3; 3"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_MATYAS_FUNCTION_HPP

src/mlpack/core/optimizers/problems/mc_cormick_function.cpp

@@ -0,0 +1,59 @@
+/**
+ * @file mc_cormick_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the McCormick function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "mc_cormick_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+McCormickFunction::McCormickFunction() { /* Nothing to do here */ }
+
+void McCormickFunction::Shuffle() { /* Nothing to do here */ }
+
+double McCormickFunction::Evaluate(const arma::mat& coordinates,
+                                const size_t /* begin */,
+                                const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  const double objective = std::sin(x1 + x2) + std::pow(x1 - x2, 2) -
+      1.5 * x1 + 2.5 * x2 + 1;
+
+  return objective;
+}
+
+double McCormickFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void McCormickFunction::Gradient(const arma::mat& coordinates,
+                                 const size_t /* begin */,
+                                 arma::mat& gradient,
+                                 const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  gradient.set_size(2, 1);
+  gradient(0) = std::cos(x1 + x2) + 2 * x1 - 2 * x2 - 1.5;
+  gradient(1) = std::cos(x1 + x2) - 2 * x1 + 2 * x2 + 2.5;
+}
+
+void McCormickFunction::Gradient(const arma::mat& coordinates,
+                                 arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/mc_cormick_function.hpp

@@ -0,0 +1,105 @@
+/**
+ * @file mc_cormick_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the McCormick function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_MC_CORMICK_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_MC_CORMICK_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The McCormick function, defined by
+ *
+ * \f[
+ * f(x) = \sin(x_1 + x_2) + (x_1 - x_2)^2 - 1.5 * x_1 + 2.5 * x_2 + 1
+ * \f]
+ *
+ * This should optimize to f(x) = -1.9133, at x = [-0.54719, -1.54719].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class McCormickFunction
+{
+ public:
+  //! Initialize the McCormickFunction.
+  McCormickFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-2; 4"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_MC_CORMICK_FUNCTION_HPP

src/mlpack/core/optimizers/problems/rastrigin_function.cpp

@@ -0,0 +1,73 @@
+/**
+ * @file rastrigin_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Rastrigin function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "rastrigin_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+RastriginFunction::RastriginFunction(const size_t n) :
+    n(n),
+    visitationOrder(arma::linspace<arma::Row<size_t> >(0, n - 1, n))
+
+{
+  initialPoint.set_size(n, 1);
+  initialPoint.fill(-3);
+}
+
+void RastriginFunction::Shuffle()
+{
+  visitationOrder = arma::shuffle(
+      arma::linspace<arma::Row<size_t> >(0, n - 1, n));
+}
+
+double RastriginFunction::Evaluate(const arma::mat& coordinates,
+                                   const size_t begin,
+                                   const size_t batchSize) const
+{
+  double objective = 0.0;
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    objective += std::pow(coordinates(p), 2) - 10.0 *
+        std::cos(2.0 * M_PI * coordinates(p));
+  }
+  objective += 10.0 * n;
+
+  return objective;
+}
+
+double RastriginFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void RastriginFunction::Gradient(const arma::mat& coordinates,
+                                 const size_t begin,
+                                 arma::mat& gradient,
+                                 const size_t batchSize) const
+{
+  gradient.zeros(n, 1);
+
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    gradient(p) += (10.0 * n) * (2 * (coordinates(p) + 10.0 * M_PI *
+        std::sin(2.0 * M_PI * coordinates(p))));
+  }
+}
+
+void RastriginFunction::Gradient(const arma::mat& coordinates,
+                                 arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/rastrigin_function.hpp

@@ -0,0 +1,117 @@
+/**
+ * @file rastrigin_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Rastrigin function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_RASTRIGIN_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_RASTRIGIN_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Rastrigin function, defined by
+ *
+ * \f[
+ * f(x) = 10 * d * \sum_{i=1}^{d} x_i^2 - 10 * \cos(2 * \pi * x_i)
+ * \f]
+ *
+ * This should optimize to f(x) = 0
+ * at x = [0, ..., 0].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {Systems of extremal control},
+ *   author  = {Rastrigin, L. A.},
+ *   journal = {Mir},
+ *   year    = {1974}
+ * }
+ * @endcode
+ */
+class RastriginFunction
+{
+ public:
+  /*
+   * Initialize the RastriginFunction.
+   *
+   * @param n Number of dimensions for the function.
+   */
+  RastriginFunction(const size_t n);
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return n; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return initialPoint; }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+ private:
+  //! Number of dimensions for the function.
+  size_t n;
+
+  //! For shuffling.
+  arma::Row<size_t> visitationOrder;
+
+  //! Initial starting point.
+  arma::mat initialPoint;
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_RASTRIGIN_FUNCTION_HPP

src/mlpack/core/optimizers/problems/rosenbrock_function.cpp

@@ -0,0 +1,60 @@
+/**
+ * @file rosenbrock_function.cpp
+ * @author Ryan Curtin
+ * @author Marcus Edel
+ *
+ * Implementation of the Rosenbrock function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "rosenbrock_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+RosenbrockFunction::RosenbrockFunction() { /* Nothing to do here */ }
+
+void RosenbrockFunction::Shuffle() { /* Nothing to do here */ }
+
+double RosenbrockFunction::Evaluate(const arma::mat& coordinates,
+                                    const size_t /* begin */,
+                                    const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  const double objective = /* f1(x) */ 100 * std::pow(x2 - std::pow(x1, 2), 2) +
+                           /* f2(x) */ std::pow(1 - x1, 2);
+
+  return objective;
+}
+
+double RosenbrockFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void RosenbrockFunction::Gradient(const arma::mat& coordinates,
+                                  const size_t /* begin */,
+                                  arma::mat& gradient,
+                                  const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+
+  gradient.set_size(2, 1);
+  gradient(0) = -2 * (1 - x1) + 400 * (std::pow(x1, 3) - x2 * x1);
+  gradient(1) = 200 * (x2 - std::pow(x1, 2));
+}
+
+void RosenbrockFunction::Gradient(const arma::mat& coordinates,
+                                  arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, 1);
+}

src/mlpack/core/optimizers/problems/rosenbrock_function.hpp

@@ -0,0 +1,108 @@
+/**
+ * @file rosenbrock_function.hpp
+ * @author Ryan Curtin
+ * @author Marcus Edel
+ *
+ * Definition of the Rosenbrock function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_ROSENBROCK_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_ROSENBROCK_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Rosenbrock function, defined by:
+ *
+ *  f(x) = f1(x) + f2(x)
+ *  f1(x) = 100 (x2 - x1^2)^2
+ *  f2(x) = (1 - x1)^2
+ *  x_0 = [-1.2, 1]
+ *
+ * This should optimize to f(x) = 0, at x = [1, 1].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Rosenbrock1960,
+ *    title  = {An Automatic Method for Finding the Greatest or Least Value of a
+ *              Function},
+ *   author  = {Rosenbrock, H. H.},
+ *   journal = {The Computer Journal},
+ *   number  = {3},
+ *   pages   = {175--184},
+ *   year    = {1960},
+ * }
+ * @endcode
+ */
+class RosenbrockFunction
+{
+ public:
+  //! Initialize the RosenbrockFunction.
+  RosenbrockFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-1.2; 1"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_ROSENBROCK_FUNCTION_HPP

src/mlpack/core/optimizers/problems/rosenbrock_wood_function.cpp

@@ -0,0 +1,64 @@
+/**
+ * @file rosenbrock_wood_function.cpp
+ * @author Ryan Curtin
+ * @author Marcus Edel
+ *
+ * Implementation of the Rosenbrock-Wood function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "rosenbrock_wood_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+RosenbrockWoodFunction::RosenbrockWoodFunction() : rf(4), wf()
+{
+  initialPoint.set_size(4, 2);
+  initialPoint.col(0) = rf.GetInitialPoint();
+  initialPoint.col(1) = wf.GetInitialPoint();
+}
+
+void RosenbrockWoodFunction::Shuffle() { /* Nothing to do here */ }
+
+double RosenbrockWoodFunction::Evaluate(const arma::mat& coordinates,
+                                        const size_t /* begin */,
+                                        const size_t /* batchSize */) const
+{
+  const double objective = rf.Evaluate(coordinates.col(0)) +
+      wf.Evaluate(coordinates.col(1));
+
+  return objective;
+}
+
+double RosenbrockWoodFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void RosenbrockWoodFunction::Gradient(const arma::mat& coordinates,
+                                      const size_t /* begin */,
+                                      arma::mat& gradient,
+                                      const size_t /* batchSize */) const
+{
+  gradient.set_size(4, 2);
+
+  arma::vec grf(4);
+  arma::vec gwf(4);
+
+  rf.Gradient(coordinates.col(0), grf);
+  wf.Gradient(coordinates.col(1), gwf);
+
+  gradient.col(0) = grf;
+  gradient.col(1) = gwf;
+}
+
+void RosenbrockWoodFunction::Gradient(const arma::mat& coordinates,
+                                      arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, 1);
+}

src/mlpack/core/optimizers/problems/rosenbrock_wood_function.hpp

@@ -0,0 +1,101 @@
+/**
+ * @file rosenbrock_wood_function.hpp
+ * @author Ryan Curtin
+ * @author Marcus Edel
+ *
+ * Definition of the Rosenbrock-Wood function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_ROSENBROCK_WOOD_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_ROSENBROCK_WOOD_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include <mlpack/core/optimizers/problems/generalized_rosenbrock_function.hpp>
+#include <mlpack/core/optimizers/problems/wood_function.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Generalized Rosenbrock function in 4 dimensions with the Wood Function in
+ * four dimensions.  In this function we are actually optimizing a 2x4 matrix of
+ * coordinates, not a vector.
+ */
+class RosenbrockWoodFunction
+{
+ public:
+  //! Initialize the RosenbrockWoodFunction.
+  RosenbrockWoodFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  const arma::mat& GetInitialPoint() const { return initialPoint; }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+ private:
+  //! Locally-stored initial point.
+  arma::mat initialPoint;
+
+  //! Locally-stored Generalized-Rosenbrock function.
+  GeneralizedRosenbrockFunction rf;
+
+  //! Locally-stored Wood function.
+  WoodFunction wf;
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_ROSENBROCK_WOOD_FUNCTION_HPP

src/mlpack/core/optimizers/problems/schwefel_function.cpp

@@ -0,0 +1,74 @@
+/**
+ * @file schwefel_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Schwefel function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "schwefel_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+SchwefelFunction::SchwefelFunction(const size_t n) :
+    n(n),
+    visitationOrder(arma::linspace<arma::Row<size_t> >(0, n - 1, n))
+
+{
+  initialPoint.set_size(n, 1);
+  initialPoint.fill(-300);
+}
+
+void SchwefelFunction::Shuffle()
+{
+  visitationOrder = arma::shuffle(
+      arma::linspace<arma::Row<size_t> >(0, n - 1, n));
+}
+
+double SchwefelFunction::Evaluate(const arma::mat& coordinates,
+                                  const size_t begin,
+                                  const size_t batchSize) const
+{
+  double objective = 0.0;
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    objective += coordinates(p) * std::sin(std::sqrt(std::abs(coordinates(p))));
+  }
+  objective -= 418.9829 * batchSize;
+
+  return objective;
+}
+
+double SchwefelFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void SchwefelFunction::Gradient(const arma::mat& coordinates,
+                                const size_t begin,
+                                arma::mat& gradient,
+                                const size_t batchSize) const
+{
+  gradient.zeros(n, 1);
+
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    gradient(p) += (std::pow(coordinates(p), 2) *
+        std::cos(std::sqrt(std::abs(coordinates(p)))) /
+        (2 * std::pow(std::abs(coordinates(p)), 1.5)) +
+        std::sin(std::sqrt(std::abs(coordinates(p)))));
+  }
+}
+
+void SchwefelFunction::Gradient(const arma::mat& coordinates,
+                                arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/schwefel_function.hpp

@@ -0,0 +1,117 @@
+/**
+ * @file schwefel_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Schwefel function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_SCHWEFEL_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_SCHWEFEL_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Schwefel function, defined by
+ *
+ * \f[
+ * f(x) = 418.9829 * d * \sum_{i=1}^{d} x_i * \sin(\sqrt(\left|x\right|))
+ * \f]
+ *
+ * This should optimize to f(x) = 0
+ * at x = [420.9687, ..., 420.9687].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {Systems of extremal control},
+ *   author  = {Rastrigin, L. A.},
+ *   journal = {Mir},
+ *   year    = {1974}
+ * }
+ * @endcode
+ */
+class SchwefelFunction
+{
+ public:
+  /*
+   * Initialize the SchwefelFunction.
+   *
+   * @param n Number of dimensions for the function.
+   */
+  SchwefelFunction(const size_t n);
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return n; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return initialPoint; }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+ private:
+  //! Number of dimensions for the function.
+  size_t n;
+
+  //! For shuffling.
+  arma::Row<size_t> visitationOrder;
+
+  //! Initial starting point.
+  arma::mat initialPoint;
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_SCHWEFEL_FUNCTION_HPP

src/mlpack/core/optimizers/problems/sgd_test_function.cpp

@@ -1,5 +1,5 @@
 /**
- * @file test_function.cpp
+ * @file sgd_test_function.cpp
  * @author Ryan Curtin
  *
  * Implementation of very simple test function for stochastic gradient descent
@@ -10,7 +10,7 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#include "test_function.hpp"
+#include "sgd_test_function.hpp"
 
 using namespace mlpack;
 using namespace mlpack::optimization;

src/mlpack/core/optimizers/problems/sgd_test_function.hpp

@@ -1,5 +1,5 @@
 /**
- * @file test_function.hpp
+ * @file sgd_test_function.hpp
  * @author Ryan Curtin
  *
  * Very simple test function for SGD.
@@ -9,8 +9,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_OPTIMIZERS_SGD_TEST_FUNCTION_HPP
-#define MLPACK_CORE_OPTIMIZERS_SGD_TEST_FUNCTION_HPP
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_SGD_TEST_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_SGD_TEST_FUNCTION_HPP
 
 #include <mlpack/prereqs.hpp>
 
@@ -45,7 +45,7 @@ class SGDTestFunction
   //! Evaluate a function.
   double Evaluate(const arma::mat& coordinates, const size_t i) const;
 
-  //! Evaluate a function for a particular batch-size
+  //! Evaluate a function for a particular batch-size.
   double Evaluate(const arma::mat& coordinates,
                   const size_t begin,
                   const size_t batchSize) const;

src/mlpack/core/optimizers/problems/sphere_function.cpp

@@ -0,0 +1,76 @@
+/**
+ * @file sphere_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Sphere function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "sphere_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+SphereFunction::SphereFunction(const size_t n) :
+    n(n),
+    visitationOrder(arma::linspace<arma::Row<size_t> >(0, n - 1, n))
+
+{
+  initialPoint.set_size(n, 1);
+
+  for (size_t i = 0; i < n; ++i) // Set to [-3.12 3.33 -3.12 3.33...].
+  {
+    if (i % 2 == 1)
+      initialPoint(i) = 5;
+    else
+      initialPoint(i) = -5;
+  }
+}
+
+void SphereFunction::Shuffle()
+{
+  visitationOrder = arma::shuffle(
+      arma::linspace<arma::Row<size_t> >(0, n - 1, n));
+}
+
+double SphereFunction::Evaluate(const arma::mat& coordinates,
+                                const size_t begin,
+                                const size_t batchSize) const
+{
+  double objective = 0.0;
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    objective += std::pow(coordinates(p), 2);
+  }
+
+  return objective;
+}
+
+double SphereFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void SphereFunction::Gradient(const arma::mat& coordinates,
+                              const size_t begin,
+                              arma::mat& gradient,
+                              const size_t batchSize) const
+{
+  gradient.zeros(n, 1);
+
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    gradient(p) += 2.0 * coordinates[p];
+  }
+}
+
+void SphereFunction::Gradient(const arma::mat& coordinates, arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/sphere_function.hpp

@@ -0,0 +1,118 @@
+/**
+ * @file sphere_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Sphere function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_SPHERE_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_SPHERE_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Sphere function, defined by
+ *
+ * \f[
+ * f(x) = \sum_{i=1}^{d} x_i^2
+ * \f]
+ *
+ * This should optimize to f(x) = 0, at x = [0, ..., 0].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class SphereFunction
+{
+ public:
+  /*
+   * Initialize the SphereFunction.
+   *
+   * @param n Number of dimensions for the function.
+   */
+  SphereFunction(const size_t n);
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return n; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return initialPoint; }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+ private:
+  //! Number of dimensions for the function.
+  size_t n;
+
+  //! For shuffling.
+  arma::Row<size_t> visitationOrder;
+
+  //! Initial starting point.
+  arma::mat initialPoint;
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_SPHERE_FUNCTION_HPP

src/mlpack/core/optimizers/problems/styblinski_tang_function.cpp

@@ -0,0 +1,73 @@
+/**
+ * @file styblinski_tang_function.cpp
+ * @author Marcus Edel
+ *
+ * Implementation of the Styblinski-Tang function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "styblinski_tang_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+StyblinskiTangFunction::StyblinskiTangFunction(const size_t n) :
+    n(n),
+    visitationOrder(arma::linspace<arma::Row<size_t> >(0, n - 1, n))
+
+{
+  initialPoint.set_size(n, 1);
+  initialPoint.fill(-5);
+}
+
+void StyblinskiTangFunction::Shuffle()
+{
+  visitationOrder = arma::shuffle(
+      arma::linspace<arma::Row<size_t> >(0, n - 1, n));
+}
+
+double StyblinskiTangFunction::Evaluate(const arma::mat& coordinates,
+                                        const size_t begin,
+                                        const size_t batchSize) const
+{
+  double objective = 0.0;
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    objective += std::pow(coordinates(p), 4) - 16 *
+        std::pow(coordinates(p), 2) + 5 * coordinates(p);
+  }
+  objective /= 2;
+
+  return objective;
+}
+
+double StyblinskiTangFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void StyblinskiTangFunction::Gradient(const arma::mat& coordinates,
+                                      const size_t begin,
+                                      arma::mat& gradient,
+                                      const size_t batchSize) const
+{
+  gradient.zeros(n, 1);
+
+  for (size_t j = begin; j < begin + batchSize; ++j)
+  {
+    const size_t p = visitationOrder[j];
+    gradient(p) += 0.5 * (4 * std::pow(coordinates(p), 3) -
+        32.0 * coordinates(p) + 5.0);
+  }
+}
+
+void StyblinskiTangFunction::Gradient(const arma::mat& coordinates,
+                                      arma::mat& gradient)
+{
+  Gradient(coordinates, 0, gradient, NumFunctions());
+}

src/mlpack/core/optimizers/problems/styblinski_tang_function.hpp

@@ -0,0 +1,119 @@
+/**
+ * @file styblinski_tang_function.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Styblinski-Tang function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_STYBLINSKI_TANG_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_STYBLINSKI_TANG_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Styblinski-Tang function, defined by
+ *
+ * \f[
+ * f(x) = 0.5 * \sum_{i=1}^{d} x_i^4 - 16_i^2+5x_i
+ * \f]
+ *
+ * This should optimize to f(x) = -39.16599 * d
+ * at x = [-2.903534, ..., -2.903534].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Jamil2013,
+ *   title   = {A Literature Survey of Benchmark Functions For Global
+ *              Optimization Problems},
+ *   author  = {Momin Jamil and Xin{-}She Yang},
+ *   journal = {CoRR},
+ *   year    = {2013},
+ *   url     = {http://arxiv.org/abs/1308.4008}
+ * }
+ * @endcode
+ */
+class StyblinskiTangFunction
+{
+ public:
+  /*
+   * Initialize the StyblinskiTangFunction.
+   *
+   * @param n Number of dimensions for the function.
+   */
+  StyblinskiTangFunction(const size_t n);
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return n; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return initialPoint; }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient);
+ private:
+  //! Number of dimensions for the function.
+  size_t n;
+
+  //! For shuffling.
+  arma::Row<size_t> visitationOrder;
+
+  //! Initial starting point.
+  arma::mat initialPoint;
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_STYBLINSKI_TANG_FUNCTION_HPP

src/mlpack/core/optimizers/problems/wood_function.cpp

@@ -0,0 +1,72 @@
+/**
+ * @file wood_function.cpp
+ * @author Ryan Curtin
+ * @author Marcus Edel
+ *
+ * Implementation of the Wood function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include "wood_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+WoodFunction::WoodFunction() { /* Nothing to do here */ }
+
+void WoodFunction::Shuffle() { /* Nothing to do here */ }
+
+double WoodFunction::Evaluate(const arma::mat& coordinates,
+                              const size_t /* begin */,
+                              const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+  const double x3 = coordinates(2);
+  const double x4 = coordinates(3);
+
+  const double objective = /* f1(x) */ 100 * std::pow(x2 - std::pow(x1, 2), 2) +
+                           /* f2(x) */ std::pow(1 - x1, 2) +
+                           /* f3(x) */ 90 * std::pow(x4 - std::pow(x3, 2), 2) +
+                           /* f4(x) */ std::pow(1 - x3, 2) +
+                           /* f5(x) */ 10 * std::pow(x2 + x4 - 2, 2) +
+                           /* f6(x) */ (1.0 / 10.0) * std::pow(x2 - x4, 2);
+
+  return objective;
+}
+
+double WoodFunction::Evaluate(const arma::mat& coordinates) const
+{
+  return Evaluate(coordinates, 0, NumFunctions());
+}
+
+void WoodFunction::Gradient(const arma::mat& coordinates,
+                            const size_t /* begin */,
+                            arma::mat& gradient,
+                            const size_t /* batchSize */) const
+{
+  // For convenience; we assume these temporaries will be optimized out.
+  const double x1 = coordinates(0);
+  const double x2 = coordinates(1);
+  const double x3 = coordinates(2);
+  const double x4 = coordinates(3);
+
+  gradient.set_size(4, 1);
+  gradient(0) = 400 * (std::pow(x1, 3) - x2 * x1) - 2 * (1 - x1);
+  gradient(1) = 200 * (x2 - std::pow(x1, 2)) + 20 * (x2 + x4 - 2) +
+      (1.0 / 5.0) * (x2 - x4);
+  gradient(2) = 360 * (std::pow(x3, 3) - x4 * x3) - 2 * (1 - x3);
+  gradient(3) = 180 * (x4 - std::pow(x3, 2)) + 20 * (x2 + x4 - 2) -
+      (1.0 / 5.0) * (x2 - x4);
+}
+
+void WoodFunction::Gradient(const arma::mat& coordinates, arma::mat& gradient)
+    const
+{
+  Gradient(coordinates, 0, gradient, 1);
+}

src/mlpack/core/optimizers/problems/wood_function.hpp

@@ -0,0 +1,112 @@
+/**
+ * @file wood_function.hpp
+ * @author Ryan Curtin
+ * @author Marcus Edel
+ *
+ * Definition of the Wood function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_PROBLEMS_WOOD_FUNCTION_HPP
+#define MLPACK_CORE_OPTIMIZERS_PROBLEMS_WOOD_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+namespace test {
+
+/**
+ * The Wood function, defined by
+ *  f(x) = f1(x) + f2(x) + f3(x) + f4(x) + f5(x) + f6(x)
+ *  f1(x) = 100 (x2 - x1^2)^2
+ *  f2(x) = (1 - x1)^2
+ *  f3(x) = 90 (x4 - x3^2)^2
+ *  f4(x) = (1 - x3)^2
+ *  f5(x) = 10 (x2 + x4 - 2)^2
+ *  f6(x) = (1 / 10) (x2 - x4)^2
+ *  x_0 = [-3, -1, -3, -1]
+ *
+ * This should optimize to f(x) = 0, at x = [1, 1, 1, 1].
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Grippo1989,
+ *   title   = {A truncated Newton method with nonmonotone line search for
+ *              unconstrained optimization},
+ *   author  = {Grippo, L. and Lampariello, F. and Lucidi, S.},
+ *   journal = {Journal of Optimization Theory and Applications},
+ *   year    = {1989},
+ *   volume  = {60},
+ *   number  = {3},
+ *   pages   = {401--419},
+ * }
+ * @endcode
+ */
+class WoodFunction
+{
+ public:
+  //! Initialize the WoodFunction.
+  WoodFunction();
+
+  /**
+  * Shuffle the order of function visitation. This may be called by the
+  * optimizer.
+  */
+  void Shuffle();
+
+  //! Return 1 (the number of functions).
+  size_t NumFunctions() const { return 1; }
+
+  //! Get the starting point.
+  arma::mat GetInitialPoint() const { return arma::mat("-3; -1; -3; -1"); }
+
+  /*
+   * Evaluate a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param batchSize Number of points to process.
+   */
+  double Evaluate(const arma::mat& coordinates,
+                  const size_t begin,
+                  const size_t batchSize) const;
+
+  /*
+   * Evaluate a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   */
+  double Evaluate(const arma::mat& coordinates) const;
+
+  /*
+   * Evaluate the gradient of a function for a particular batch-size.
+   *
+   * @param coordinates The function coordinates.
+   * @param begin The first function.
+   * @param gradient The function gradient.
+   * @param batchSize Number of points to process.
+   */
+  void Gradient(const arma::mat& coordinates,
+                const size_t begin,
+                arma::mat& gradient,
+                const size_t batchSize) const;
+
+  /*
+   * Evaluate the gradient of a function with the given coordinates.
+   *
+   * @param coordinates The function coordinates.
+   * @param gradient The function gradient.
+   */
+  void Gradient(const arma::mat& coordinates, arma::mat& gradient) const;
+};
+
+} // namespace test
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_PROBLEMS_WOOD_FUNCTION_HPP

src/mlpack/core/optimizers/sgd/CMakeLists.txt

@@ -5,8 +5,6 @@ set(SOURCES
   update_policies/vanilla_update.hpp
   sgd.hpp
   sgd_impl.hpp
-  test_function.hpp
-  test_function.cpp
 )
 
 set(DIR_SRCS)

src/mlpack/core/util/mlpack_main.hpp

@@ -74,6 +74,7 @@ int main(int argc, char** argv)
 
 #include <mlpack/bindings/tests/test_option.hpp>
 #include <mlpack/bindings/tests/ignore_check.hpp>
+#include <mlpack/bindings/tests/clean_memory.hpp>
 
 // These functions will do nothing.
 #define PRINT_PARAM_STRING(A) std::string(" ")
@@ -138,6 +139,10 @@ static const std::string testName = "";
 
 PARAM_FLAG("verbose", "Display informational messages and the full list of "
     "parameters and timers at the end of execution.", "v");
+PARAM_FLAG("copy_all_inputs", "If specified, all input parameters will be deep"
+    " copied before the method is run.  This is useful for debugging problems "
+    "where the input parameters are being modified by the algorithm, but can "
+    "slow down the code.", "");
 
 // Nothing else needs to be defined---the binding will use mlpackMain() as-is.
 

src/mlpack/core/util/param.hpp

@@ -1058,9 +1058,9 @@ using DatasetInfo = DatasetMapper<IncrementPolicy, std::string>;
   // There are no uses of required models, so that is not an option to this
   // macro (it would be easy to add).
   #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \
-      static mlpack::util::Option<TYPE> \
+      static mlpack::util::Option<TYPE*> \
       JOIN(cli_option_dummy_model_, __COUNTER__) \
-      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, testName);
+      (nullptr, ID, DESC, ALIAS, #TYPE, REQ, IN, false, testName);
 #else
   // We have to do some really bizarre stuff since __COUNTER__ isn't defined. I
   // don't think we can absolutely guarantee success, but it should be "good
@@ -1113,9 +1113,9 @@ using DatasetInfo = DatasetMapper<IncrementPolicy, std::string>;
       !TRANS, testName);
 
   #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \
-      static mlpack::util::Option<TYPE> \
+      static mlpack::util::Option<TYPE*> \
       JOIN(JOIN(cli_option_dummy_object_model_, __LINE__), opt) \
-      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, \
+      (nullptr, ID, DESC, ALIAS, #TYPE, REQ, IN, false, \
       testName);
 #endif
 

src/mlpack/methods/adaboost/adaboost_main.cpp

@@ -145,10 +145,11 @@ static void mlpackMain()
 
   ReportIgnoredParam({{ "test", false }}, "output");
 
-  AdaBoostModel m;
+  AdaBoostModel* m;
   if (CLI::HasParam("training"))
   {
     mat trainingData = std::move(CLI::GetParam<arma::mat>("training"));
+    m = new AdaBoostModel();
 
     // Load labels.
     arma::Row<size_t> labelsIn;
@@ -172,28 +173,28 @@ static void mlpackMain()
     Row<size_t> labels;
 
     // Normalize the labels.
-    data::NormalizeLabels(labelsIn, labels, m.Mappings());
+    data::NormalizeLabels(labelsIn, labels, m->Mappings());
 
     // Get other training parameters.
     const double tolerance = CLI::GetParam<double>("tolerance");
     const size_t iterations = (size_t) CLI::GetParam<int>("iterations");
     const string weakLearner = CLI::GetParam<string>("weak_learner");
     if (weakLearner == "decision_stump")
-      m.WeakLearnerType() = AdaBoostModel::WeakLearnerTypes::DECISION_STUMP;
+      m->WeakLearnerType() = AdaBoostModel::WeakLearnerTypes::DECISION_STUMP;
     else if (weakLearner == "perceptron")
-      m.WeakLearnerType() = AdaBoostModel::WeakLearnerTypes::PERCEPTRON;
+      m->WeakLearnerType() = AdaBoostModel::WeakLearnerTypes::PERCEPTRON;
 
-    const size_t numClasses = m.Mappings().n_elem;
+    const size_t numClasses = m->Mappings().n_elem;
     Log::Info << numClasses << " classes in dataset." << endl;
 
     Timer::Start("adaboost_training");
-    m.Train(trainingData, labels, numClasses, iterations, tolerance);
+    m->Train(trainingData, labels, numClasses, iterations, tolerance);
     Timer::Stop("adaboost_training");
   }
   else
   {
     // We have a specified input model.
-    m = std::move(CLI::GetParam<AdaBoostModel>("input_model"));
+    m = CLI::GetParam<AdaBoostModel*>("input_model");
   }
 
   // Perform classification, if desired.
@@ -201,24 +202,21 @@ static void mlpackMain()
   {
     mat testingData = std::move(CLI::GetParam<arma::mat>("test"));
 
-    if (testingData.n_rows != m.Dimensionality())
+    if (testingData.n_rows != m->Dimensionality())
       Log::Fatal << "Test data dimensionality (" << testingData.n_rows << ") "
           << "must be the same as the model dimensionality ("
-          << m.Dimensionality() << ")!" << endl;
+          << m->Dimensionality() << ")!" << endl;
 
     Row<size_t> predictedLabels(testingData.n_cols);
     Timer::Start("adaboost_classification");
-    m.Classify(testingData, predictedLabels);
+    m->Classify(testingData, predictedLabels);
     Timer::Stop("adaboost_classification");
 
     Row<size_t> results;
-    data::RevertLabels(predictedLabels, m.Mappings(), results);
+    data::RevertLabels(predictedLabels, m->Mappings(), results);
 
-    if (CLI::HasParam("output"))
-      CLI::GetParam<arma::Row<size_t>>("output") = std::move(results);
+    CLI::GetParam<arma::Row<size_t>>("output") = std::move(results);
   }
 
-  // Should we save the model, too?
-  if (CLI::HasParam("output_model"))
-    CLI::GetParam<AdaBoostModel>("output_model") = std::move(m);
+  CLI::GetParam<AdaBoostModel*>("output_model") = m;
 }

src/mlpack/methods/ann/layer/CMakeLists.txt

@@ -65,6 +65,8 @@ set(SOURCES
   recurrent_attention_impl.hpp
   reinforce_normal.hpp
   reinforce_normal_impl.hpp
+  sigmoid_cross_entropy_error.hpp
+  sigmoid_cross_entropy_error_impl.hpp
   select.hpp
   select_impl.hpp
   sequential.hpp

src/mlpack/methods/ann/layer/layer_types.hpp

@@ -33,6 +33,7 @@
 #include <mlpack/methods/ann/layer/mean_pooling.hpp>
 #include <mlpack/methods/ann/layer/parametric_relu.hpp>
 #include <mlpack/methods/ann/layer/reinforce_normal.hpp>
+#include <mlpack/methods/ann/layer/sigmoid_cross_entropy_error.hpp>
 #include <mlpack/methods/ann/layer/select.hpp>
 
 // Convolution modules.
@@ -116,6 +117,7 @@ using LayerTypes = boost::variant<
     Recurrent<arma::mat, arma::mat>*,
     RecurrentAttention<arma::mat, arma::mat>*,
     ReinforceNormal<arma::mat, arma::mat>*,
+    SigmoidCrossEntropyError<arma::mat, arma::mat>*,
     Select<arma::mat, arma::mat>*,
     Sequential<arma::mat, arma::mat>*,
     VRClassReward<arma::mat, arma::mat>*

src/mlpack/methods/ann/layer/log_softmax_impl.hpp

@@ -32,8 +32,8 @@ void LogSoftMax<InputDataType, OutputDataType>::Forward(
   arma::mat maxInput = arma::repmat(arma::max(input), input.n_rows, 1);
   output = (maxInput - input);
 
-  // Approximation of the hyperbolic tangent. The acuracy however is
-  // about 0.00001 lower as using tanh. Credits go to Leon Bottou.
+  // Approximation of the base-e exponential function. The acuracy however is
+  // about 0.00001 lower as using exp. Credits go to Leon Bottou.
   output.transform([](double x)
   {
     //! Fast approximation of exp(-x) for x positive.