From 3b64a54f3417cf17c0c0207881f760ac2e44f818 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Thu, 22 Jun 2017 14:53:54 +0530 Subject: [PATCH 01/41] Initial implementation of parallel SGD --- .../optimizers/parallel_sgd/parallel_sgd.hpp | 83 +++++++++++++++++++ .../parallel_sgd/parallel_sgd_impl.hpp | 50 +++++++++++ .../parallel_sgd/sparse_svm_function.hpp | 18 ++++ .../stepsize_policies/constant_step.hpp | 31 +++++++ .../stepsize_policies/exponential_backoff.hpp | 74 +++++++++++++++++ 5 files changed, 256 insertions(+) create mode 100644 src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp create mode 100644 src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp create mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp create mode 100644 src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp create mode 100644 src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/exponential_backoff.hpp diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp new file mode 100644 index 0000000000..6225ea79f8 --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -0,0 +1,83 @@ +/** + * @file parallel_sgd.hpp + * @author Shikhar Bhardwaj + * + * Parallel Stochastic Gradient Descent. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_HPP + +#include + +#include "stepsize_policies/constant_step.hpp" + +// TODO FIXME : Documentation + +namespace mlpack { +namespace optimization { + +/** + * An implementation of parallel stochastic gradient descent using the lock-free + * HOGWILD! approach. + * + * For more information, see the following. + * @misc{1106.5730, + * Author = {Feng Niu and Benjamin Recht and Christopher Re and Stephen J. + * Wright}, + * Title = {HOGWILD!: A Lock-Free Approach to Parallelizing Stochastic Gradient + * Descent}, + * Year = {2011}, + * Eprint = {arXiv:1106.5730}, + * } + */ +template < + typename SparseFunctionType, + typename StepsizePolicyType = ConstantStep +> +class ParallelSGD +{ + public: + ParallelSGD(SparseFunctionType& function, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const StepsizePolicyType stepPolicy = StepsizePolicyType()); + + double Optimize(SparseFunctionType& function, arma::mat& iterate); + + double Optimize(arma::mat& iterate) + { + return Optimize(this->function, iterate); + } + + //! Get the instantiated function to be optimized. + const SparseFunctionType& Function() const { return function; } + //! Modify the instantiated function. + SparseFunctionType& Function() { return function; } + //! Get the instantiated function to be optimized. + size_t MaxIterations() const { return maxIterations; } + //! Modify the instantiated function. + size_t& MaxIterations() { return maxIterations; } + //! Get the instantiated function to be optimized. + double Tolerance() const { return tolerance; } + //! Modify the instantiated function. + double& Tolerance() { return tolerance; } + + private: + SparseFunctionType& function; + size_t maxIterations; + double tolerance; + StepsizePolicyType& stepPolicy; +}; + +} +} + +// Include implementation. +#include "parallel_sgd_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp new file mode 100644 index 0000000000..51389a1974 --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -0,0 +1,50 @@ +/** + * @file parallel_sgd_impl.hpp + * @author Shikhar Bhardwaj + * + * Implementation of Parallel Stochastic Gradient Descent. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_IMPL_HPP + +#include + +#include "stepsize_policies/constant_step.hpp" + +// In case it hasn't been included yet. +#include "parallel_sgd.hpp" + +namespace mlpack { +namespace optimization { + +template +ParallelSGD::ParallelSGD( + SparseFunctionType& function, + const size_t maxIterations, + const double tolerance, + const StepsizePolicyType stepPolicy) : + function(function), + maxIterations(maxIterations), + tolerance(tolerance), + stepPolicy(stepPolicy) +{ /* Nothing to do. */ } + +template +double ParallelSGD::Optimize( + SparseFunctionType& function, + arma::mat& iterate) +{ +} + +} +} + +// Include implementation. +#include "parallel_sgd_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp new file mode 100644 index 0000000000..5273733706 --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -0,0 +1,18 @@ +/** + * @file parallel_sgd_impl.hpp + * @author Shikhar Bhardwaj + * + * Implementation of the hinge loss function for training a sparse SVM with the + * parallel SGD algorithm + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +class SparseSVMLoss{ + public: + SparseSVMLoss(arma::mat& dataset, ) +}; diff --git a/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp b/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp new file mode 100644 index 0000000000..5f673ca3e4 --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp @@ -0,0 +1,31 @@ +/** + * @file constant_step.hpp + * @author Shikhar Bhardwaj + * + * Constant step size policy for parallel Stochastic Gradient Descent. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_CONSTANT_STEP_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_CONSTANT_STEP_HPP + +#include + +namespace mlpack{ +namespace optimization{ +class ConstantStep{ + public: + ConstantStep(double initalStep) : step(initalStep) {} + double getStepSize(size_t /* n_epoch */){ + return step; + } + private: + double step; +}; +} +} + +#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/exponential_backoff.hpp new file mode 100644 index 0000000000..3badc58f71 --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/exponential_backoff.hpp @@ -0,0 +1,74 @@ +/** + * @file constant_step.hpp + * @author Shikhar Bhardwaj + * + * Exponential backoff step size policy for parallel Stochastic Gradient + * Descent. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_CONSTANT_STEP_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_CONSTANT_STEP_HPP + +#include + +namespace mlpack{ +namespace optimization{ + +/** + * Exponential backoff stepsize reduction policy for parallel SGD. + * + * For more information, see the following. + * + * @misc{1106.5730, + * Author = {Feng Niu and Benjamin Recht and Christopher Re and Stephen J. + * Wright}, + * Title = {HOGWILD!: A Lock-Free Approach to Parallelizing Stochastic + * Gradient Descent}, + * Year = {2011}, + * Eprint = {arXiv:1106.5730}, + * } + * + * This stepsize update scheme gives robust 1/k convergence rates to the + * implementation of parallel SGD. + */ +class ExponentialBackoff{ + public: + /** + * Construct the exponential backoff policy with the required parameters. + * + * @param firstBackoffEpoch The number of updates to run before the first + * stepsize backoff. + * @param step The initial stepsize(gamma). + * @param beta The reduction factor. + */ + ExponentialBackoff(size_t firstBackoffEpoch, double step, double beta) : + firstBackoffEpoch(firstBackoffEpoch), step(step), beta(beta) + { + cutoffEpoch = firstBackoffEpoch; + } + /** + * Get the step size for the current gradient update. + * + * @param n_epoch The iteration number of the current update. + */ + double getStepSize(size_t n_epoch) + { + if (n_epoch >= cutoffEpoch) + { + step /= beta; + cutoffEpoch += firstBackoffEpoch / beta; + } + return step; + } + private: + size_t firstBackoffEpoch, cutoffEpoch; + double step, beta; +}; +} +} + +#endif From b4b21116b2a8bdbf89852c3e7a94a62ac2863683 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Thu, 22 Jun 2017 23:02:38 +0530 Subject: [PATCH 02/41] Initial implementation of the sparse svm loss function for parallel SGD --- .../parallel_sgd/parallel_sgd_impl.hpp | 22 ++++++ .../parallel_sgd/sparse_svm_function.hpp | 20 +++++- .../parallel_sgd/sparse_svm_function_impl.hpp | 70 +++++++++++++++++++ .../stepsize_policies/constant_step.hpp | 2 +- 4 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 51389a1974..104c230db9 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -39,6 +39,28 @@ double ParallelSGD::Optimize( SparseFunctionType& function, arma::mat& iterate) { + for (size_t i = 0; i < maxIterations; ++i){ + double stepSize = stepPolicy.StepSize(i); + function.Initialize(); + #pragma omp parallel + { + // Each processor gets a subset of the instances + arma::Col instances = + function.RandomInstanceSet(omp_get_thread_num(), omp_get_num_threads()); + for (size_t i = 0; i < instances.n_elem; ++i){ + // Each instance affects only some components of the decision variable + arma::Col components = function.Components(instances[i]); + // Evaluate the gradient + // FIXME: Should evaluate only at the components required + arma::vec gradient = function.Gradient(iterate, instances[i]); + + for(size_t j = 0; j < components.n_elem; ++i){ + #pragma omp atomic + iterate[components[j]] -= stepSize * gradient[components[j]]; + } + } + } + } } } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp index 5273733706..59cb481526 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -10,9 +10,25 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_SVM_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_SVM_HPP #include -class SparseSVMLoss{ +class SparseSVMLossFunction{ public: - SparseSVMLoss(arma::mat& dataset, ) + SparseSVMLossFunction(arma::mat& dataset, arma::vec& labels); + arma::vec Gradient(arma::mat& weights, size_t component); + arma::Col VisitationOrder(size_t thread_id, size_t max_threads); + arma::Col Components(size_t id); + void GenerateVisitationOrder(); + private: + arma::mat dataset; + arma::vec labels; + arma::Col visitationOrder; + size_t numFunctions; }; + +// Include implementation +#include "sparse_svm_function_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp new file mode 100644 index 0000000000..0bd44a58cf --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -0,0 +1,70 @@ +/** + * @file parallel_sgd_impl.hpp + * @author Shikhar Bhardwaj + * + * Implementation of the hinge loss function for training a sparse SVM with the + * parallel SGD algorithm + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_SVM_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_SVM_IMPL_HPP + +// In case it hasn't been included yet. +#include "sparse_svm_function.hpp" + +SparseSVMLossFunction::SparseSVMLossFunction( + arma::mat& dataset, arma::vec& labels) : dataset(dataset), labels(labels) +{ + numFunctions = dataset.n_cols; + GenerateVisitationOrder(); +} + +void SparseSVMLossFunction::GenerateVisitationOrder() +{ + visitationOrder = arma::shuffle(arma::linspace>(0, + (numFunctions - 1), numFunctions)); +} + +arma::Col SparseSVMLossFunction::VisitationOrder( + size_t thread_id, size_t max_threads) +{ + arma::Col threadShare; + if (thread_id == max_threads - 1){ + // The last thread gets the remaining instances + threadShare = visitationOrder.subvec(thread_id * (numFunctions / + max_threads), numFunctions - 1); + } + else + { + // An equal distribution of data + threadShare = visitationOrder.subvec(thread_id * (numFunctions / + max_threads), (thread_id + 1) * (numFunctions / max_threads) - 1); + } + return threadShare; +} + +arma::vec SparseSVMLossFunction::Gradient( + arma::mat& weights, size_t id) +{ + double dot = 1 - labels(id) * arma::dot(weights, dataset.unsafe_col(id)); + return (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : + -1 * weights labels(id); +} +arma::Col SparseSVMLossFunction::Components(size_t id) +{ + std::vector nonZeroComponents; + for(size_t i = 0; i < dataset.n_rows; ++i) + { + if(dataset(i, id) != 0.f) + { + nonZeroComponents.push_back(i); + } + } + return arma::Col(nonZeroComponents); +} + +#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp b/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp index 5f673ca3e4..b66dd8f6ff 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp @@ -19,7 +19,7 @@ namespace optimization{ class ConstantStep{ public: ConstantStep(double initalStep) : step(initalStep) {} - double getStepSize(size_t /* n_epoch */){ + double StepSize(size_t /* n_epoch */){ return step; } private: From 9380e25825ca9222bd3ad7c4aa63e2ea1b05090f Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Thu, 22 Jun 2017 23:25:21 +0530 Subject: [PATCH 03/41] Terminate parallel SGD when objective is within tolerance --- .../parallel_sgd/parallel_sgd_impl.hpp | 19 +++++++++++++++---- .../parallel_sgd/sparse_svm_function.hpp | 1 + .../parallel_sgd/sparse_svm_function_impl.hpp | 11 ++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 104c230db9..c0fe514445 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -39,19 +39,23 @@ double ParallelSGD::Optimize( SparseFunctionType& function, arma::mat& iterate) { - for (size_t i = 0; i < maxIterations; ++i){ + double overallObjective = 0; + double lastObjective = DBL_MAX; + + for (size_t i = 1; i != maxIterations; ++i){ + overallObjective = 0; double stepSize = stepPolicy.StepSize(i); - function.Initialize(); + function.GenerateVisitationOrder(); #pragma omp parallel { // Each processor gets a subset of the instances arma::Col instances = - function.RandomInstanceSet(omp_get_thread_num(), omp_get_num_threads()); + function.RandomInstanceSet(omp_get_thread_num(), + omp_get_num_threads()); for (size_t i = 0; i < instances.n_elem; ++i){ // Each instance affects only some components of the decision variable arma::Col components = function.Components(instances[i]); // Evaluate the gradient - // FIXME: Should evaluate only at the components required arma::vec gradient = function.Gradient(iterate, instances[i]); for(size_t j = 0; j < components.n_elem; ++i){ @@ -60,7 +64,14 @@ double ParallelSGD::Optimize( } } } + // Evaluate the function + overallObjective = function.Evaluate(iterate); + if(std::abs(overallObjective - lastObjective) < tolerance){ + return overallObjective; + } + lastObjective = overallObjective; } + return overallObjective; } } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp index 59cb481526..53e58ecdae 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -17,6 +17,7 @@ class SparseSVMLossFunction{ public: SparseSVMLossFunction(arma::mat& dataset, arma::vec& labels); + double Evaluate(arma::mat &weights); arma::vec Gradient(arma::mat& weights, size_t component); arma::Col VisitationOrder(size_t thread_id, size_t max_threads); arma::Col Components(size_t id); diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp index 0bd44a58cf..56419368a3 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -23,6 +23,15 @@ SparseSVMLossFunction::SparseSVMLossFunction( GenerateVisitationOrder(); } +double SparseSVMLossFunction::Evaluate(arma::mat& weights) +{ + double eval = 0; + for(size_t i = 0; i < numFunctions; ++i){ + eval += std::max(0.0, 1 - labels(i) * arma::dot(weights, dataset.col(i))); + } + return eval; +} + void SparseSVMLossFunction::GenerateVisitationOrder() { visitationOrder = arma::shuffle(arma::linspace>(0, @@ -52,7 +61,7 @@ arma::vec SparseSVMLossFunction::Gradient( { double dot = 1 - labels(id) * arma::dot(weights, dataset.unsafe_col(id)); return (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : - -1 * weights labels(id); + (-1 * dataset.unsafe_col(id) * labels(id)); } arma::Col SparseSVMLossFunction::Components(size_t id) { From 02555873230ee4c22b21d60f61a228fee8dcc602 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sat, 24 Jun 2017 21:09:46 +0530 Subject: [PATCH 04/41] Update SparseFunctionType API and parallel_sgd documentation --- .../optimizers/parallel_sgd/parallel_sgd.hpp | 114 ++++++++++++++++-- .../parallel_sgd/parallel_sgd_impl.hpp | 53 ++++++-- .../parallel_sgd/sparse_svm_function.hpp | 7 +- .../parallel_sgd/sparse_svm_function_impl.hpp | 39 +----- 4 files changed, 155 insertions(+), 58 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 6225ea79f8..ef255fa518 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -16,8 +16,6 @@ #include "stepsize_policies/constant_step.hpp" -// TODO FIXME : Documentation - namespace mlpack { namespace optimization { @@ -34,21 +32,93 @@ namespace optimization { * Year = {2011}, * Eprint = {arXiv:1106.5730}, * } + * + * For Parallel SGD to work, a SparseFunctionType template parameter is + * required. This class must implement the following functions: + * + * size_t NumFunctions(); + * double Evaluate(const arma::mat& coordinates, const size_t i); + * void Gradient(const arma::mat& coordinates, + * const size_t i, + * arma::mat& gradient); + * arma::Col Components(size_t id); + * + * In these functions the parameter id refers to which individual function (or + * gradient) is being evaluated. In case of a data-dependent function, the id + * would refer to the index of the datapoint(or training example). + * The data is distributed uniformly among the threads made available to the + * program by the OpenMP runtime. + * The class is expected to implement a Components function, which takes in the + * index of a datapoint and returns a list of component indices(of the decision + * variable) for which the decision variable needs to be updated. + * + * @tparam SparseFunctionType Sparse, Decomposable objective function type to be + * minimized. + * @tparam DecayPolicyType Step size update policy used by parallel SGD + * to update the stepsize after each iteration. */ template < typename SparseFunctionType, - typename StepsizePolicyType = ConstantStep + typename DecayPolicyType > class ParallelSGD { public: + /** + * Construct the parallel SGD optimizer to optimize the given function with + * the given parameters. One iteration means one batch of datapoints processed + * by each thread. The default values given here are just for reference, it is + * suggested that the values are set according to the task at hand. + * + * @param function Function to be optimized(minimized). + * @param maxIterations Maximum number of iterations allowed. + * @param batchSize Number of datapoints to be processed in one iteration by + * each thread. + * @param tolerance Maximum absolute tolerance to terminate the algorithm. + * @param decayPolicy The step size update policy to use. + */ ParallelSGD(SparseFunctionType& function, - const size_t maxIterations = 100000, + const size_t maxIterations = 100, + const size_t batchSize = 10000, const double tolerance = 1e-5, - const StepsizePolicyType stepPolicy = StepsizePolicyType()); + const DecayPolicyType decayPolicy = DecayPolicyType()); + /** + * Generate the indices to be visited by each thread before iteration. + * Generates a randomly shuffled vector of datapoint indices (range 0 to + * function.NumFunctions()). + */ + void GenerateVisitationOrder(); + + /** + * Get the share of datapoint indices to be updated by the thread with given + * thread id. + * + * @param thread_id The id of the current thread. Range 0-OMP_NUM_THREADS. + * @return Vector of datapoint indices to be visited by the current thread. + */ + arma::Col ThreadShare(size_t thread_id); + + /** + * Optimize the given function using the parallel SGD algorithm. The given + * starting point will be modified to store the finishing point of the + * algorithm, and the value of the loss function at the final point is + * returned. + * + * @param function Function to be opmtimized(minimized). + * @param iterate Starting point(will be modified). + * @return Objective value at the final point. + */ double Optimize(SparseFunctionType& function, arma::mat& iterate); + /** + * Optimize the given function using stochastic gradient descent. The given + * starting point will be modified to store the finishing point of the + * algorithm, and the final objective value is returned. + * + * @param iterate Starting point (will be modified). + * @return Objective value of the final point. + */ double Optimize(arma::mat& iterate) { return Optimize(this->function, iterate); @@ -58,20 +128,44 @@ class ParallelSGD const SparseFunctionType& Function() const { return function; } //! Modify the instantiated function. SparseFunctionType& Function() { return function; } - //! Get the instantiated function to be optimized. + + //! Get the maximum number of iterations (0 indicates no limits). size_t MaxIterations() const { return maxIterations; } - //! Modify the instantiated function. + //! Modify the maximum number of iterations (0 indicates no limits). size_t& MaxIterations() { return maxIterations; } - //! Get the instantiated function to be optimized. + + //! Get the number of datapoints to be processed in one iteration by each + //! thread. + size_t BatchSize() const { return batchSize; } + //! Modify the number of datapoints to be processed in one iteration by each + //! thread. + size_t& BatchSize() { return batchSize; } + + //! Get the tolerance for termination. double Tolerance() const { return tolerance; } - //! Modify the instantiated function. + //! Modify the tolerance for termination. double& Tolerance() { return tolerance; } + //! Get the step size decay policy. + DecayPolicyType& DecayPolicy() const { return decayPolicy; } + //! Modify the step size decay policy. + DecayPolicyType& DecayPolicy() { return decayPolicy; } + private: + //! The instantiated function. SparseFunctionType& function; + + //! The maximum number of allowed iterations. size_t maxIterations; + + //! The number of datapoints to be processed in one iteration by each thread. + size_t batchSize; + + //! The tolerance for termination. double tolerance; - StepsizePolicyType& stepPolicy; + + //! The step size decay policy. + DecayPolicyType& decayPolicy; }; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index c0fe514445..c1891d42eb 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -34,6 +34,36 @@ ParallelSGD::ParallelSGD( stepPolicy(stepPolicy) { /* Nothing to do. */ } +template +void ParallelSGD< + SparseFunctionType, + StepsizePolicyType>::GenerateVisitationOrder() +{ + visitationOrder = arma::shuffle(arma::linspace>(0, + (function.numFunctions() - 1), function.NumFunctions())); +} + +template +arma::Col ParallelSGD< + SparseFunctionType, + StepsizePolicyType>::ThreadShare(size_t thread_id, size_t max_threads) +{ + arma::Col threadShare; + size_t examplesPerThread = std::floor(function.NumFunctions() / max_threads); + if (thread_id == max_threads - 1){ + // The last thread gets the remaining instances + threadShare = visitationOrder.subvec(thread_id * examplesPerThread, + function.numFunctions() - 1); + } + else + { + // An equal distribution of data + threadShare = visitationOrder.subvec(thread_id * examplesPerThread, + (thread_id + 1) * examplesPerThread - 1); + } + return threadShare; +} + template double ParallelSGD::Optimize( SparseFunctionType& function, @@ -45,27 +75,32 @@ double ParallelSGD::Optimize( for (size_t i = 1; i != maxIterations; ++i){ overallObjective = 0; double stepSize = stepPolicy.StepSize(i); - function.GenerateVisitationOrder(); + GenerateVisitationOrder(); #pragma omp parallel { // Each processor gets a subset of the instances - arma::Col instances = - function.RandomInstanceSet(omp_get_thread_num(), + arma::Col instances = ThreadShare(omp_get_thread_num(), omp_get_num_threads()); - for (size_t i = 0; i < instances.n_elem; ++i){ + for (size_t j = 0; j < instances.n_elem; ++j) + { // Each instance affects only some components of the decision variable - arma::Col components = function.Components(instances[i]); + arma::Col components = function.Components(instances[j]); // Evaluate the gradient - arma::vec gradient = function.Gradient(iterate, instances[i]); + arma::vec gradient; + function.Gradient(iterate, instances[j], gradient); - for(size_t j = 0; j < components.n_elem; ++i){ + for(size_t k = 0; k < components.n_elem; ++k) + { #pragma omp atomic - iterate[components[j]] -= stepSize * gradient[components[j]]; + iterate[components[k]] -= stepSize * gradient[components[k]]; } } } // Evaluate the function - overallObjective = function.Evaluate(iterate); + overallObjective = 0; + for(size_t j = 0; j < function.NumFunctions(); ++j){ + overallObjective += function.Evaluate(iterate, j); + } if(std::abs(overallObjective - lastObjective) < tolerance){ return overallObjective; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp index 53e58ecdae..91034f6181 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -17,15 +17,12 @@ class SparseSVMLossFunction{ public: SparseSVMLossFunction(arma::mat& dataset, arma::vec& labels); - double Evaluate(arma::mat &weights); - arma::vec Gradient(arma::mat& weights, size_t component); - arma::Col VisitationOrder(size_t thread_id, size_t max_threads); + double Evaluate(arma::mat &weights, size_t id); + void Gradient(arma::mat& weights, size_t id, arma::mat& gradient); arma::Col Components(size_t id); - void GenerateVisitationOrder(); private: arma::mat dataset; arma::vec labels; - arma::Col visitationOrder; size_t numFunctions; }; diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp index 56419368a3..d408dba655 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -20,47 +20,18 @@ SparseSVMLossFunction::SparseSVMLossFunction( arma::mat& dataset, arma::vec& labels) : dataset(dataset), labels(labels) { numFunctions = dataset.n_cols; - GenerateVisitationOrder(); } -double SparseSVMLossFunction::Evaluate(arma::mat& weights) +double SparseSVMLossFunction::Evaluate(arma::mat& weights, size_t id) { - double eval = 0; - for(size_t i = 0; i < numFunctions; ++i){ - eval += std::max(0.0, 1 - labels(i) * arma::dot(weights, dataset.col(i))); - } - return eval; + return std::max(0.0, 1 - labels(id) * arma::dot(weights, dataset.col(id))); } -void SparseSVMLossFunction::GenerateVisitationOrder() -{ - visitationOrder = arma::shuffle(arma::linspace>(0, - (numFunctions - 1), numFunctions)); -} - -arma::Col SparseSVMLossFunction::VisitationOrder( - size_t thread_id, size_t max_threads) -{ - arma::Col threadShare; - if (thread_id == max_threads - 1){ - // The last thread gets the remaining instances - threadShare = visitationOrder.subvec(thread_id * (numFunctions / - max_threads), numFunctions - 1); - } - else - { - // An equal distribution of data - threadShare = visitationOrder.subvec(thread_id * (numFunctions / - max_threads), (thread_id + 1) * (numFunctions / max_threads) - 1); - } - return threadShare; -} - -arma::vec SparseSVMLossFunction::Gradient( - arma::mat& weights, size_t id) +void SparseSVMLossFunction::Gradient( + arma::mat& weights, size_t id, arma::mat& gradient) { double dot = 1 - labels(id) * arma::dot(weights, dataset.unsafe_col(id)); - return (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : + gradient = (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : (-1 * dataset.unsafe_col(id) * labels(id)); } arma::Col SparseSVMLossFunction::Components(size_t id) From a1a05d4e8da26f651a4cf70066fd513012e05471 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sat, 24 Jun 2017 22:59:26 +0530 Subject: [PATCH 05/41] Refactor StepSizePolicy to be similar to minibatch SGD --- .../optimizers/parallel_sgd/CMakeLists.txt | 13 +++ .../decay_policies/CMakeLists.txt | 11 +++ .../constant_step.hpp | 0 .../exponential_backoff.hpp | 6 +- .../optimizers/parallel_sgd/parallel_sgd.hpp | 44 +++++---- .../parallel_sgd/parallel_sgd_impl.hpp | 94 +++++++++++-------- 6 files changed, 107 insertions(+), 61 deletions(-) create mode 100644 src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt create mode 100644 src/mlpack/core/optimizers/parallel_sgd/decay_policies/CMakeLists.txt rename src/mlpack/core/optimizers/parallel_sgd/{stepsize_policies => decay_policies}/constant_step.hpp (100%) rename src/mlpack/core/optimizers/parallel_sgd/{stepsize_policies => decay_policies}/exponential_backoff.hpp (92%) diff --git a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt new file mode 100644 index 0000000000..07c5dc517a --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt @@ -0,0 +1,13 @@ +set(SOURCES + parallel_sgd.hpp + parallel_sgd_impl.hpp + sparse_svm_function.hpp + sparse_svm_function_impl.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/CMakeLists.txt b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/CMakeLists.txt new file mode 100644 index 0000000000..9bf1b40964 --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/CMakeLists.txt @@ -0,0 +1,11 @@ +set(SOURCES + constant_step.hpp + exponential_backoff.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp similarity index 100% rename from src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/constant_step.hpp rename to src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp diff --git a/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp similarity index 92% rename from src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/exponential_backoff.hpp rename to src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index 3badc58f71..5c4637c938 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/stepsize_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -1,8 +1,8 @@ /** - * @file constant_step.hpp + * @file exponential_backoff.hpp * @author Shikhar Bhardwaj * - * Exponential backoff step size policy for parallel Stochastic Gradient + * Exponential backoff step size decay policy for parallel Stochastic Gradient * Descent. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -55,7 +55,7 @@ class ExponentialBackoff{ * * @param n_epoch The iteration number of the current update. */ - double getStepSize(size_t n_epoch) + double GetStepSize(size_t n_epoch) { if (n_epoch >= cutoffEpoch) { diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index ef255fa518..d44ac83b4a 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -14,8 +14,6 @@ #include -#include "stepsize_policies/constant_step.hpp" - namespace mlpack { namespace optimization { @@ -81,23 +79,7 @@ class ParallelSGD const size_t maxIterations = 100, const size_t batchSize = 10000, const double tolerance = 1e-5, - const DecayPolicyType decayPolicy = DecayPolicyType()); - - /** - * Generate the indices to be visited by each thread before iteration. - * Generates a randomly shuffled vector of datapoint indices (range 0 to - * function.NumFunctions()). - */ - void GenerateVisitationOrder(); - - /** - * Get the share of datapoint indices to be updated by the thread with given - * thread id. - * - * @param thread_id The id of the current thread. Range 0-OMP_NUM_THREADS. - * @return Vector of datapoint indices to be visited by the current thread. - */ - arma::Col ThreadShare(size_t thread_id); + const DecayPolicyType& decayPolicy = DecayPolicyType()); /** * Optimize the given function using the parallel SGD algorithm. The given @@ -152,6 +134,29 @@ class ParallelSGD DecayPolicyType& DecayPolicy() { return decayPolicy; } private: + /** + * Generate the indices to be visited by each thread before iteration. + * Generates a randomly shuffled vector of datapoint indices (range 0 to + * function.NumFunctions()). + * + * @param visitationOrder Out param with the indices of the datapoints for the + * current iteration. + */ + void GenerateVisitationOrder(arma::Col& visitationOrder); + + /** + * Get the share of datapoint indices to be updated by the thread with given + * thread id. + * + * @param thread_id The id of the current thread. Range 0-OMP_NUM_THREADS. + * @param visitationOrder The random list of datapoint indices for the current + * iteration. + * @return Vector of datapoint indices to be visited by the current thread. + */ + arma::Col ThreadShare(size_t thread_id, + const arma::Col& visitationOrder); + + //! The instantiated function. SparseFunctionType& function; @@ -166,6 +171,7 @@ class ParallelSGD //! The step size decay policy. DecayPolicyType& decayPolicy; + }; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index c1891d42eb..d6b8144271 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -14,55 +14,27 @@ #include -#include "stepsize_policies/constant_step.hpp" - // In case it hasn't been included yet. #include "parallel_sgd.hpp" namespace mlpack { namespace optimization { -template -ParallelSGD::ParallelSGD( +template +ParallelSGD::ParallelSGD( SparseFunctionType& function, const size_t maxIterations, + const size_t batchSize, const double tolerance, - const StepsizePolicyType stepPolicy) : + const DecayPolicyType& decayPolicy) : function(function), maxIterations(maxIterations), + batchSize(batchSize), tolerance(tolerance), - stepPolicy(stepPolicy) + decayPolicy(decayPolicy) { /* Nothing to do. */ } -template -void ParallelSGD< - SparseFunctionType, - StepsizePolicyType>::GenerateVisitationOrder() -{ - visitationOrder = arma::shuffle(arma::linspace>(0, - (function.numFunctions() - 1), function.NumFunctions())); -} -template -arma::Col ParallelSGD< - SparseFunctionType, - StepsizePolicyType>::ThreadShare(size_t thread_id, size_t max_threads) -{ - arma::Col threadShare; - size_t examplesPerThread = std::floor(function.NumFunctions() / max_threads); - if (thread_id == max_threads - 1){ - // The last thread gets the remaining instances - threadShare = visitationOrder.subvec(thread_id * examplesPerThread, - function.numFunctions() - 1); - } - else - { - // An equal distribution of data - threadShare = visitationOrder.subvec(thread_id * examplesPerThread, - (thread_id + 1) * examplesPerThread - 1); - } - return threadShare; -} template double ParallelSGD::Optimize( @@ -74,13 +46,18 @@ double ParallelSGD::Optimize( for (size_t i = 1; i != maxIterations; ++i){ overallObjective = 0; - double stepSize = stepPolicy.StepSize(i); - GenerateVisitationOrder(); + + // Get the stepsize for this iteration + double stepSize = decayPolicy.StepSize(i); + arma::Col visitationOrder; + GenerateVisitationOrder(visitationOrder); + #pragma omp parallel { // Each processor gets a subset of the instances + // Each subset is of size batchSize arma::Col instances = ThreadShare(omp_get_thread_num(), - omp_get_num_threads()); + visitationOrder); for (size_t j = 0; j < instances.n_elem; ++j) { // Each instance affects only some components of the decision variable @@ -96,12 +73,16 @@ double ParallelSGD::Optimize( } } } + // Evaluate the function overallObjective = 0; - for(size_t j = 0; j < function.NumFunctions(); ++j){ + for(size_t j = 0; j < function.NumFunctions(); ++j) + { overallObjective += function.Evaluate(iterate, j); } - if(std::abs(overallObjective - lastObjective) < tolerance){ + + if(std::abs(overallObjective - lastObjective) < tolerance) + { return overallObjective; } lastObjective = overallObjective; @@ -109,6 +90,41 @@ double ParallelSGD::Optimize( return overallObjective; } +template +void ParallelSGD< + SparseFunctionType, + StepsizePolicyType>::GenerateVisitationOrder( + arma::Col& visitationOrder) +{ + visitationOrder = arma::shuffle(arma::linspace>(0, + (function.NumFunctions() - 1), function.NumFunctions())); +} + +template +arma::Col ParallelSGD< + SparseFunctionType, + StepsizePolicyType>::ThreadShare(size_t thread_id, + const arma::Col& visitationOrder) +{ + if(thread_id * batchSize >= visitationOrder.n_elem) + { + // No data for this thread. + return arma::Col(); + } + else if((thread_id + 1) * batchSize >= visitationOrder.n_elem) + { + // The last few elements. + return visitationOrder.subvec(thread_id * batchSize, + visitationOrder.n_elem - 1); + } + else + { + // Equal distribution of batchSize examples to each thread. + return visitationOrder.subvec(thread_id * batchSize, + (thread_id + 1) * batchSize - 1); + } +} + } } From d767e8cd37c2fa0138d89f8df0cc8311630abeb5 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sat, 24 Jun 2017 23:18:16 +0530 Subject: [PATCH 06/41] Cleanup StepsizePolicy from parallel SGD impelementation --- .../decay_policies/exponential_backoff.hpp | 2 +- .../optimizers/parallel_sgd/parallel_sgd_impl.hpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index 5c4637c938..71e3aa646d 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -55,7 +55,7 @@ class ExponentialBackoff{ * * @param n_epoch The iteration number of the current update. */ - double GetStepSize(size_t n_epoch) + double StepSize(size_t n_epoch) { if (n_epoch >= cutoffEpoch) { diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index d6b8144271..b244d7a73f 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -36,8 +36,8 @@ ParallelSGD::ParallelSGD( -template -double ParallelSGD::Optimize( +template +double ParallelSGD::Optimize( SparseFunctionType& function, arma::mat& iterate) { @@ -90,20 +90,20 @@ double ParallelSGD::Optimize( return overallObjective; } -template +template void ParallelSGD< SparseFunctionType, - StepsizePolicyType>::GenerateVisitationOrder( + DecayPolicyType>::GenerateVisitationOrder( arma::Col& visitationOrder) { visitationOrder = arma::shuffle(arma::linspace>(0, (function.NumFunctions() - 1), function.NumFunctions())); } -template +template arma::Col ParallelSGD< SparseFunctionType, - StepsizePolicyType>::ThreadShare(size_t thread_id, + DecayPolicyType>::ThreadShare(size_t thread_id, const arma::Col& visitationOrder) { if(thread_id * batchSize >= visitationOrder.n_elem) From 0a65080f3abef37de6c82531f3652ca0220b8fb1 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sat, 24 Jun 2017 23:28:22 +0530 Subject: [PATCH 07/41] Fix style and readability issues in parallel SGD implementation --- .../decay_policies/constant_step.hpp | 5 ++-- .../decay_policies/exponential_backoff.hpp | 8 ++++--- .../optimizers/parallel_sgd/parallel_sgd.hpp | 11 ++++----- .../parallel_sgd/parallel_sgd_impl.hpp | 23 ++++++++----------- .../parallel_sgd/sparse_svm_function_impl.hpp | 8 +++---- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp index b66dd8f6ff..d62076057e 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp @@ -25,7 +25,8 @@ class ConstantStep{ private: double step; }; -} -} + +} // namespace optimization +} // namespace mlpack #endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index 71e3aa646d..3d6458b178 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -45,7 +45,7 @@ class ExponentialBackoff{ * @param step The initial stepsize(gamma). * @param beta The reduction factor. */ - ExponentialBackoff(size_t firstBackoffEpoch, double step, double beta) : + ExponentialBackoff(size_t firstBackoffEpoch, double step, double beta) : firstBackoffEpoch(firstBackoffEpoch), step(step), beta(beta) { cutoffEpoch = firstBackoffEpoch; @@ -64,11 +64,13 @@ class ExponentialBackoff{ } return step; } + private: size_t firstBackoffEpoch, cutoffEpoch; double step, beta; }; -} -} + +} // namespace optimization +} // namespace mlpack #endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index d44ac83b4a..59482872cc 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -75,8 +75,8 @@ class ParallelSGD * @param tolerance Maximum absolute tolerance to terminate the algorithm. * @param decayPolicy The step size update policy to use. */ - ParallelSGD(SparseFunctionType& function, - const size_t maxIterations = 100, + ParallelSGD(SparseFunctionType& function, + const size_t maxIterations = 100, const size_t batchSize = 10000, const double tolerance = 1e-5, const DecayPolicyType& decayPolicy = DecayPolicyType()); @@ -162,7 +162,7 @@ class ParallelSGD //! The maximum number of allowed iterations. size_t maxIterations; - + //! The number of datapoints to be processed in one iteration by each thread. size_t batchSize; @@ -171,11 +171,10 @@ class ParallelSGD //! The step size decay policy. DecayPolicyType& decayPolicy; - }; -} -} +} // namespace optimization +} // namespace mlpack // Include implementation. #include "parallel_sgd_impl.hpp" diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index b244d7a73f..341702c246 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -22,8 +22,8 @@ namespace optimization { template ParallelSGD::ParallelSGD( - SparseFunctionType& function, - const size_t maxIterations, + SparseFunctionType& function, + const size_t maxIterations, const size_t batchSize, const double tolerance, const DecayPolicyType& decayPolicy) : @@ -38,7 +38,7 @@ ParallelSGD::ParallelSGD( template double ParallelSGD::Optimize( - SparseFunctionType& function, + SparseFunctionType& function, arma::mat& iterate) { double overallObjective = 0; @@ -66,7 +66,7 @@ double ParallelSGD::Optimize( arma::vec gradient; function.Gradient(iterate, instances[j], gradient); - for(size_t k = 0; k < components.n_elem; ++k) + for (size_t k = 0; k < components.n_elem; ++k) { #pragma omp atomic iterate[components[k]] -= stepSize * gradient[components[k]]; @@ -76,12 +76,12 @@ double ParallelSGD::Optimize( // Evaluate the function overallObjective = 0; - for(size_t j = 0; j < function.NumFunctions(); ++j) + for (size_t j = 0; j < function.NumFunctions(); ++j) { overallObjective += function.Evaluate(iterate, j); } - if(std::abs(overallObjective - lastObjective) < tolerance) + if (std::abs(overallObjective - lastObjective) < tolerance) { return overallObjective; } @@ -106,12 +106,12 @@ arma::Col ParallelSGD< DecayPolicyType>::ThreadShare(size_t thread_id, const arma::Col& visitationOrder) { - if(thread_id * batchSize >= visitationOrder.n_elem) + if (thread_id * batchSize >= visitationOrder.n_elem) { // No data for this thread. return arma::Col(); } - else if((thread_id + 1) * batchSize >= visitationOrder.n_elem) + else if ((thread_id + 1) * batchSize >= visitationOrder.n_elem) { // The last few elements. return visitationOrder.subvec(thread_id * batchSize, @@ -125,10 +125,7 @@ arma::Col ParallelSGD< } } -} -} - -// Include implementation. -#include "parallel_sgd_impl.hpp" +} // namespace optimization +} // namespace mlpack #endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp index d408dba655..541f841f8f 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -31,18 +31,16 @@ void SparseSVMLossFunction::Gradient( arma::mat& weights, size_t id, arma::mat& gradient) { double dot = 1 - labels(id) * arma::dot(weights, dataset.unsafe_col(id)); - gradient = (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : + gradient = (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : (-1 * dataset.unsafe_col(id) * labels(id)); } arma::Col SparseSVMLossFunction::Components(size_t id) { std::vector nonZeroComponents; - for(size_t i = 0; i < dataset.n_rows; ++i) + for (size_t i = 0; i < dataset.n_rows; ++i) { - if(dataset(i, id) != 0.f) - { + if (dataset(i, id) != 0.f) nonZeroComponents.push_back(i); - } } return arma::Col(nonZeroComponents); } From d7453184ee811671fc643e807b179594d0da8ab0 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sun, 25 Jun 2017 22:40:26 +0530 Subject: [PATCH 08/41] Refactor Sparse SVM for sparse matrices --- .../decay_policies/exponential_backoff.hpp | 4 ++-- .../parallel_sgd/sparse_svm_function.hpp | 19 +++++++++++----- .../parallel_sgd/sparse_svm_function_impl.hpp | 22 +++++++++++-------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index 3d6458b178..d233de6688 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_CONSTANT_STEP_HPP -#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_CONSTANT_STEP_HPP +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_EXP_BACKOFF_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_EXP_BACKOFF_HPP #include diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp index 91034f6181..b77814fa23 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -16,14 +16,23 @@ class SparseSVMLossFunction{ public: - SparseSVMLossFunction(arma::mat& dataset, arma::vec& labels); - double Evaluate(arma::mat &weights, size_t id); - void Gradient(arma::mat& weights, size_t id, arma::mat& gradient); + SparseSVMLossFunction() = default; + SparseSVMLossFunction(arma::SpMat& dataset, arma::vec& labels); + double Evaluate(arma::vec& weights, size_t id); + void Gradient(arma::vec& weights, size_t id, arma::mat& gradient); arma::Col Components(size_t id); + + const arma::SpMat& Dataset() const { return dataset; } + arma::SpMat& Dataset() { return dataset; } + + const arma::vec& Labels() const { return labels; } + arma::vec& Labels() { return labels; } + + size_t NumFunctions(); + private: - arma::mat dataset; + arma::SpMat dataset; arma::vec labels; - size_t numFunctions; }; // Include implementation diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp index 541f841f8f..8f6f671055 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -17,22 +17,21 @@ #include "sparse_svm_function.hpp" SparseSVMLossFunction::SparseSVMLossFunction( - arma::mat& dataset, arma::vec& labels) : dataset(dataset), labels(labels) -{ - numFunctions = dataset.n_cols; -} + arma::SpMat& dataset, arma::vec& labels) : + dataset(dataset), labels(labels) +{ /* Nothing to do */ } -double SparseSVMLossFunction::Evaluate(arma::mat& weights, size_t id) +double SparseSVMLossFunction::Evaluate(arma::vec& weights, size_t id) { - return std::max(0.0, 1 - labels(id) * arma::dot(weights, dataset.col(id))); + return std::max(0.0, 1 - labels(id) * arma::dot(dataset.col(id), weights)); } void SparseSVMLossFunction::Gradient( - arma::mat& weights, size_t id, arma::mat& gradient) + arma::vec& weights, size_t id, arma::mat& gradient) { - double dot = 1 - labels(id) * arma::dot(weights, dataset.unsafe_col(id)); + double dot = 1 - labels(id) * arma::dot(weights, dataset.col(id)); gradient = (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : - (-1 * dataset.unsafe_col(id) * labels(id)); + (-1 * dataset.col(id) * labels(id)); } arma::Col SparseSVMLossFunction::Components(size_t id) { @@ -44,5 +43,10 @@ arma::Col SparseSVMLossFunction::Components(size_t id) } return arma::Col(nonZeroComponents); } +size_t SparseSVMLossFunction::NumFunctions() +{ + return dataset.n_cols; +} + #endif From d5a920af39c7947f8b804b574768bdf3170240d0 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 26 Jun 2017 01:50:38 +0530 Subject: [PATCH 09/41] Sparse SVM Example run complete --- src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp | 2 +- .../core/optimizers/parallel_sgd/parallel_sgd_impl.hpp | 6 ++++++ .../core/optimizers/parallel_sgd/sparse_svm_function.hpp | 4 ++-- .../optimizers/parallel_sgd/sparse_svm_function_impl.hpp | 6 +++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 59482872cc..f416e387ab 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -170,7 +170,7 @@ class ParallelSGD double tolerance; //! The step size decay policy. - DecayPolicyType& decayPolicy; + DecayPolicyType decayPolicy; }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 341702c246..c8e96865ee 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -44,6 +44,8 @@ double ParallelSGD::Optimize( double overallObjective = 0; double lastObjective = DBL_MAX; + Log::Info << std::endl; + for (size_t i = 1; i != maxIterations; ++i){ overallObjective = 0; @@ -81,11 +83,15 @@ double ParallelSGD::Optimize( overallObjective += function.Evaluate(iterate, j); } + Log::Info << "\nObjective : " << overallObjective << " Iteration : " << i; if (std::abs(overallObjective - lastObjective) < tolerance) { + Log::Info << "\n Parallel SGD terminated with objective : " + << overallObjective << std::endl; return overallObjective; } lastObjective = overallObjective; + std::flush(std::cout); } return overallObjective; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp index b77814fa23..ef38f7090e 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -18,8 +18,8 @@ class SparseSVMLossFunction{ public: SparseSVMLossFunction() = default; SparseSVMLossFunction(arma::SpMat& dataset, arma::vec& labels); - double Evaluate(arma::vec& weights, size_t id); - void Gradient(arma::vec& weights, size_t id, arma::mat& gradient); + double Evaluate(const arma::vec& weights, size_t id); + void Gradient(const arma::vec& weights, size_t id, arma::mat& gradient); arma::Col Components(size_t id); const arma::SpMat& Dataset() const { return dataset; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp index 8f6f671055..154ca7cb67 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -21,17 +21,17 @@ SparseSVMLossFunction::SparseSVMLossFunction( dataset(dataset), labels(labels) { /* Nothing to do */ } -double SparseSVMLossFunction::Evaluate(arma::vec& weights, size_t id) +double SparseSVMLossFunction::Evaluate(const arma::vec& weights, size_t id) { return std::max(0.0, 1 - labels(id) * arma::dot(dataset.col(id), weights)); } void SparseSVMLossFunction::Gradient( - arma::vec& weights, size_t id, arma::mat& gradient) + const arma::vec& weights, size_t id, arma::mat& gradient) { double dot = 1 - labels(id) * arma::dot(weights, dataset.col(id)); gradient = (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : - (-1 * dataset.col(id) * labels(id)); + (-1 * arma::vec(dataset.col(id) * labels(id))); } arma::Col SparseSVMLossFunction::Components(size_t id) { From d0577f436e6497ddc80afcbc2ba6634a1809155a Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 26 Jun 2017 14:29:03 +0530 Subject: [PATCH 10/41] Use sparse matrix iterators to speed up Components function in SparseSVMLossFunction --- .../parallel_sgd/decay_policies/exponential_backoff.hpp | 3 ++- .../core/optimizers/parallel_sgd/parallel_sgd_impl.hpp | 2 +- .../optimizers/parallel_sgd/sparse_svm_function_impl.hpp | 7 ++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index d233de6688..60e9fef56b 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -54,12 +54,13 @@ class ExponentialBackoff{ * Get the step size for the current gradient update. * * @param n_epoch The iteration number of the current update. + * @return The stepsize for the current iteration. */ double StepSize(size_t n_epoch) { if (n_epoch >= cutoffEpoch) { - step /= beta; + step *= beta; cutoffEpoch += firstBackoffEpoch / beta; } return step; diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index c8e96865ee..57defcb276 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -86,7 +86,7 @@ double ParallelSGD::Optimize( Log::Info << "\nObjective : " << overallObjective << " Iteration : " << i; if (std::abs(overallObjective - lastObjective) < tolerance) { - Log::Info << "\n Parallel SGD terminated with objective : " + Log::Info << "\n Parallel SGD terminated with objective : " << overallObjective << std::endl; return overallObjective; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp index 154ca7cb67..95e9de6aa5 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -33,16 +33,17 @@ void SparseSVMLossFunction::Gradient( gradient = (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : (-1 * arma::vec(dataset.col(id) * labels(id))); } + arma::Col SparseSVMLossFunction::Components(size_t id) { std::vector nonZeroComponents; - for (size_t i = 0; i < dataset.n_rows; ++i) + for (auto cur = dataset.begin_col(id); cur != dataset.end_col(id); ++cur) { - if (dataset(i, id) != 0.f) - nonZeroComponents.push_back(i); + nonZeroComponents.push_back(cur.row()); } return arma::Col(nonZeroComponents); } + size_t SparseSVMLossFunction::NumFunctions() { return dataset.n_cols; From cf066f001a5dfaab95cd5d4bd6a9552104b187c7 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 26 Jun 2017 17:49:53 +0530 Subject: [PATCH 11/41] Add tests for Parallel SGD --- .../optimizers/parallel_sgd/parallel_sgd.hpp | 6 +- .../parallel_sgd/parallel_sgd_impl.hpp | 15 ++--- .../parallel_sgd/sparse_svm_function.hpp | 19 +++++- .../parallel_sgd/sparse_test_function.hpp | 66 +++++++++++++++++++ .../sparse_test_function_impl.hpp | 48 ++++++++++++++ 5 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp create mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index f416e387ab..780a9a0877 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -76,9 +76,9 @@ class ParallelSGD * @param decayPolicy The step size update policy to use. */ ParallelSGD(SparseFunctionType& function, - const size_t maxIterations = 100, - const size_t batchSize = 10000, - const double tolerance = 1e-5, + const size_t maxIterations, + const size_t batchSize, + const double tolerance, const DecayPolicyType& decayPolicy = DecayPolicyType()); /** diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 57defcb276..a1111559cb 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -12,8 +12,6 @@ #ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_IMPL_HPP #define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_IMPL_HPP -#include - // In case it hasn't been included yet. #include "parallel_sgd.hpp" @@ -44,13 +42,12 @@ double ParallelSGD::Optimize( double overallObjective = 0; double lastObjective = DBL_MAX; - Log::Info << std::endl; - - for (size_t i = 1; i != maxIterations; ++i){ + for (size_t i = 1; i <= maxIterations; ++i){ overallObjective = 0; // Get the stepsize for this iteration double stepSize = decayPolicy.StepSize(i); + arma::Col visitationOrder; GenerateVisitationOrder(visitationOrder); @@ -63,6 +60,7 @@ double ParallelSGD::Optimize( for (size_t j = 0; j < instances.n_elem; ++j) { // Each instance affects only some components of the decision variable + // TODO: SFINAE here arma::Col components = function.Components(instances[j]); // Evaluate the gradient arma::vec gradient; @@ -86,13 +84,14 @@ double ParallelSGD::Optimize( Log::Info << "\nObjective : " << overallObjective << " Iteration : " << i; if (std::abs(overallObjective - lastObjective) < tolerance) { - Log::Info << "\n Parallel SGD terminated with objective : " - << overallObjective << std::endl; + Log::Info << "\nParallel SGD terminated with objective delta " + << " within tolerance : " << overallObjective << std::endl; return overallObjective; } lastObjective = overallObjective; - std::flush(std::cout); } + Log::Info << "\n Parallel SGD terminated with objective : " + << overallObjective << std::endl; return overallObjective; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp index ef38f7090e..00f66566b4 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -16,22 +16,39 @@ class SparseSVMLossFunction{ public: - SparseSVMLossFunction() = default; + //! Nothing to do for the default constructor. + SparseSVMLossFunction(); + + //! Member initialization constructor. SparseSVMLossFunction(arma::SpMat& dataset, arma::vec& labels); + + //! Evaluate a function. double Evaluate(const arma::vec& weights, size_t id); + + //! Evaluate the gradient of a function. void Gradient(const arma::vec& weights, size_t id, arma::mat& gradient); + + //! Get the list of non-zero components of the gradient of a function. arma::Col Components(size_t id); + //! Get the dataset. const arma::SpMat& Dataset() const { return dataset; } + //! Modify the dataset. arma::SpMat& Dataset() { return dataset; } + //! Get the labels. const arma::vec& Labels() const { return labels; } + //! Modify the labels. arma::vec& Labels() { return labels; } + //! Return the number of functions. size_t NumFunctions(); private: + //! The datapoints for training. arma::SpMat dataset; + + //! The labels, y_i. arma::vec labels; }; diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp new file mode 100644 index 0000000000..2bf7485d20 --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp @@ -0,0 +1,66 @@ +/** + * @file sparse_test_function.hpp + * @author Shikhar Bhardwaj + * + * Sparse test function for Parallel SGD. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_TEST_FUNCTION_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_TEST_FUNCTION_HPP + +#include + +namespace mlpack { +namespace optimization { +namespace test { + +// A simple test function. Each dimension has a parabola with a +// distinct minima. Each update is guaranteed to be sparse(only a single +// dimension is updated in the decision variable by each thread). At the end of +// a reasonable number of iterations, each value in the decision variable should +// be at the vertex of the parabola in that dimension. +class SparseTestFunction +{ + public: + //! Nothing to do for the constructor. + SparseTestFunction() { } + + //! Return 6 (the number of functions). + size_t NumFunctions() const { return 4; } + + //! Get the starting point. + arma::mat GetInitialPoint() const { return arma::mat("0; 0; 0; 0;"); } + + //! Get the list of non-zero components of the gradient of a function. + arma::Col Components(size_t id); + + //! Evaluate a function. + double Evaluate(const arma::mat& coordinates, const size_t i) const; + + //! Evaluate the gradient of a function. + void Gradient(const arma::mat& coordinates, + const size_t i, + arma::mat& gradient) const; + private: + // Each quadratic polynomial is monic. The intercept and coefficient of the + // first order term is stored. + + //! The vector storing the intercepts + arma::vec intercepts = {20, 12, 15, 100}; + + //! The vector having coefficients of the first order term + arma::vec bi = {-4, -2, -3, -8}; +}; + +} // namespace test +} // namespace optimization +} // namespace mlpack + +// Include implementation +#include "sparse_test_function_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp new file mode 100644 index 0000000000..55144c55c9 --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp @@ -0,0 +1,48 @@ +/** + * @file sparse_test_function.hpp + * @author Shikhar Bhardwaj + * + * Sparse test function for Parallel SGD. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_TEST_FUNCTION_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_TEST_FUNCTION_IMPL_HPP + +// In case it hasn't been included yet. +#include "sparse_test_function.hpp" + +namespace mlpack { +namespace optimization { +namespace test { + +//! Evaluate a function. +double SparseTestFunction::Evaluate( + const arma::mat& coordinates, const size_t i) const +{ + return coordinates[i] * coordinates[i] + bi[i] * coordinates[i] + + intercepts[i]; +} + + //! Evaluate the gradient of a function. +void SparseTestFunction::Gradient(const arma::mat& coordinates, + const size_t i, + arma::mat& gradient) const +{ + gradient = arma::vec(coordinates.n_rows, 1, arma::fill::zeros); + gradient[i] = 2 * coordinates[i] + bi[i]; +} + +arma::Col SparseTestFunction::Components(size_t id) +{ + return arma::Col({ id }); +} + +} // namespace test +} // namespace optimization +} // namespace mlpack + +#endif From 37470437ec690b8f44d29fd464081ddf6340b8c2 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 26 Jun 2017 18:19:14 +0530 Subject: [PATCH 12/41] Tests for ParallelSGD and ExponentialBackoff --- src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/parallel_sgd_test.cpp | 76 ++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 src/mlpack/tests/parallel_sgd_test.cpp diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 97998c76c9..4a9096c5bd 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -67,6 +67,7 @@ add_executable(mlpack_test nmf_test.cpp nystroem_method_test.cpp octree_test.cpp + parallel_sgd_test.cpp pca_test.cpp perceptron_test.cpp q_learning_test.cpp diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp new file mode 100644 index 0000000000..0654f20bfc --- /dev/null +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -0,0 +1,76 @@ +/** + * @file parallel_sgd_test.cpp + * @author Shikhar Bhardwaj + * + * Test file for Parallel SGD. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace std; +using namespace arma; +using namespace mlpack; +using namespace mlpack::optimization; +using namespace mlpack::optimization::test; + +BOOST_AUTO_TEST_SUITE(ParallelSGDTest); + +/** + * Test the correctness of the Parallel SGD implementation using a specified + * sparse test function, with guaranteed disjoint updates between different + * threads. + */ +BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) +{ + SparseTestFunction f; + + ConstantStep decayPolicy(0.1); + + ParallelSGD s(f, 10000, 1, 1e-5, + decayPolicy); + + arma::mat coordinates = f.GetInitialPoint(); + double result = s.Optimize(coordinates); + + // The final value of the objective funtion should be close to the optimal + // value, that is the sum of values at the vertices of the parabolae. + BOOST_REQUIRE_CLOSE(result, 123.75, 0.01); + + // The co-ordinates should be the vertices of the parabolae. + BOOST_REQUIRE_CLOSE(coordinates[0], 2, 0.1); + BOOST_REQUIRE_CLOSE(coordinates[1], 1, 0.1); + BOOST_REQUIRE_CLOSE(coordinates[2], 1.5, 0.1); + BOOST_REQUIRE_CLOSE(coordinates[3], 4, 0.1); +} + +/** + * Test the correctness of the Exponential backoff stepsize decay policy. + */ +BOOST_AUTO_TEST_CASE(ExponentialBackoffDecayTest) +{ + ExponentialBackoff decayPolicy(100, 100, 0.9); + + // At the first iteration, the decay should be unchanged + BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(1), 100); + // At the 99th iteration + BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(99), 100); + // At the 100th iteration, decay should be changed + BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(100), 90); + // At the 210th iteration, decay should be unchanged + BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(210), 90); + // At the 211th iteration, decay should be changed + BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(211), 81); +} + +BOOST_AUTO_TEST_SUITE_END(); From f5c3cb2c94ecd37f0447145055dd6d92222959d9 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 26 Jun 2017 19:30:52 +0530 Subject: [PATCH 13/41] Update comments in decay policies and adjust test parameters in ParallelSGDTest --- .../decay_policies/constant_step.hpp | 18 +- .../decay_policies/exponential_backoff.hpp | 13 +- .../optimizers/parallel_sgd/parallel_sgd.hpp | 6 +- .../parallel_sgd/parallel_sgd_impl.hpp | 3 - src/mlpack/tests/CMakeLists.txt | 218 +++++++++--------- src/mlpack/tests/parallel_sgd_test.cpp | 20 +- 6 files changed, 149 insertions(+), 129 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp index d62076057e..f43876bfde 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp @@ -16,13 +16,27 @@ namespace mlpack{ namespace optimization{ + +/** + * Implementation of the ConstantStep stepsize decay policy for parallel SGD. + */ class ConstantStep{ public: - ConstantStep(double initalStep) : step(initalStep) {} - double StepSize(size_t /* n_epoch */){ + ConstantStep(double initalStep) : step(initalStep) { /* Nothing to do */ } + + /** + * This function is called in each iteration before the gradient update. + * + * @param n_epoch The iteration number for which the stepsize is to be + * calculated. + * @return The step size for the current iteration. + */ + double StepSize(size_t /* n_epoch */) + { return step; } private: + //! The initial stepsize, which remains unchanged double step; }; diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index 60e9fef56b..d797a01205 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -67,8 +67,17 @@ class ExponentialBackoff{ } private: - size_t firstBackoffEpoch, cutoffEpoch; - double step, beta; + //! The first iteration at which the stepsize should be reduced. + size_t firstBackoffEpoch; + + //! The iteration at which the next decay will be performed. + size_t cutoffEpoch; + + //! The initial stepsize. + double step; + + //! The reduction factor, should be in range (0, 1). + double beta; }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 780a9a0877..340c72144c 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -13,6 +13,7 @@ #define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_HPP #include +#include namespace mlpack { namespace optimization { @@ -65,8 +66,7 @@ class ParallelSGD /** * Construct the parallel SGD optimizer to optimize the given function with * the given parameters. One iteration means one batch of datapoints processed - * by each thread. The default values given here are just for reference, it is - * suggested that the values are set according to the task at hand. + * by each thread. * * @param function Function to be optimized(minimized). * @param maxIterations Maximum number of iterations allowed. @@ -79,7 +79,7 @@ class ParallelSGD const size_t maxIterations, const size_t batchSize, const double tolerance, - const DecayPolicyType& decayPolicy = DecayPolicyType()); + const DecayPolicyType& decayPolicy); /** * Optimize the given function using the parallel SGD algorithm. The given diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index a1111559cb..9b0b17735d 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -32,8 +32,6 @@ ParallelSGD::ParallelSGD( decayPolicy(decayPolicy) { /* Nothing to do. */ } - - template double ParallelSGD::Optimize( SparseFunctionType& function, @@ -60,7 +58,6 @@ double ParallelSGD::Optimize( for (size_t j = 0; j < instances.n_elem; ++j) { // Each instance affects only some components of the decision variable - // TODO: SFINAE here arma::Col components = function.Components(instances[j]); // Evaluate the gradient arma::vec gradient; diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 4a9096c5bd..4df1839c1c 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -1,116 +1,116 @@ # mlpack test executable. add_executable(mlpack_test - activation_functions_test.cpp - adaboost_test.cpp - adam_test.cpp - ada_delta_test.cpp - ada_grad_test.cpp - akfn_test.cpp - aknn_test.cpp - ann_layer_test.cpp - arma_extend_test.cpp - armadillo_svd_test.cpp - aug_lagrangian_test.cpp - binarize_test.cpp - block_krylov_svd_test.cpp - cf_test.cpp - cli_test.cpp - convolution_test.cpp - convolutional_network_test.cpp - cosine_tree_test.cpp - cv_test.cpp - dbscan_test.cpp - decision_stump_test.cpp - decision_tree_test.cpp - det_test.cpp - distribution_test.cpp - drusilla_select_test.cpp - emst_test.cpp - fastmks_test.cpp - feedforward_network_test.cpp - gmm_test.cpp - gradient_descent_test.cpp - hmm_test.cpp - hoeffding_tree_test.cpp - hyperplane_test.cpp - imputation_test.cpp - ind2sub_test.cpp - init_rules_test.cpp - kernel_test.cpp - kernel_pca_test.cpp - kernel_traits_test.cpp - kfn_test.cpp - kmeans_test.cpp - knn_test.cpp - krann_search_test.cpp - ksinit_test.cpp - lars_test.cpp - lbfgs_test.cpp - lin_alg_test.cpp - linear_regression_test.cpp - load_save_test.cpp - local_coordinate_coding_test.cpp - log_test.cpp - logistic_regression_test.cpp - lrsdp_test.cpp - lsh_test.cpp - math_test.cpp - matrix_completion_test.cpp - maximal_inputs_test.cpp - mean_shift_test.cpp - metric_test.cpp - minibatch_sgd_test.cpp + #activation_functions_test.cpp + #adaboost_test.cpp + #adam_test.cpp + #ada_delta_test.cpp + #ada_grad_test.cpp + #akfn_test.cpp + #aknn_test.cpp + #ann_layer_test.cpp + #arma_extend_test.cpp + #armadillo_svd_test.cpp + #aug_lagrangian_test.cpp + #binarize_test.cpp + #block_krylov_svd_test.cpp + #cf_test.cpp + #cli_test.cpp + #convolution_test.cpp + #convolutional_network_test.cpp + #cosine_tree_test.cpp + #cv_test.cpp + #dbscan_test.cpp + #decision_stump_test.cpp + #decision_tree_test.cpp + #det_test.cpp + #distribution_test.cpp + #drusilla_select_test.cpp + #emst_test.cpp + #fastmks_test.cpp + #feedforward_network_test.cpp + #gmm_test.cpp + #gradient_descent_test.cpp + #hmm_test.cpp + #hoeffding_tree_test.cpp + #hyperplane_test.cpp + #imputation_test.cpp + #ind2sub_test.cpp + #init_rules_test.cpp + #kernel_test.cpp + #kernel_pca_test.cpp + #kernel_traits_test.cpp + #kfn_test.cpp + #kmeans_test.cpp + #knn_test.cpp + #krann_search_test.cpp + #ksinit_test.cpp + #lars_test.cpp + #lbfgs_test.cpp + #lin_alg_test.cpp + #linear_regression_test.cpp + #load_save_test.cpp + #local_coordinate_coding_test.cpp + #log_test.cpp + #logistic_regression_test.cpp + #lrsdp_test.cpp + #lsh_test.cpp + #math_test.cpp + #matrix_completion_test.cpp + #maximal_inputs_test.cpp + #mean_shift_test.cpp + #metric_test.cpp + #minibatch_sgd_test.cpp mlpack_test.cpp - momentum_sgd_test.cpp - nbc_test.cpp - nca_test.cpp - nmf_test.cpp - nystroem_method_test.cpp - octree_test.cpp + #momentum_sgd_test.cpp + #nbc_test.cpp + #nca_test.cpp + #nmf_test.cpp + #nystroem_method_test.cpp + #octree_test.cpp parallel_sgd_test.cpp - pca_test.cpp - perceptron_test.cpp - q_learning_test.cpp - qdafn_test.cpp - quic_svd_test.cpp - radical_test.cpp - randomized_svd_test.cpp - range_search_test.cpp - recurrent_network_test.cpp - rectangle_tree_test.cpp - regularized_svd_test.cpp - rl_components_test.cpp - rmsprop_test.cpp - sa_test.cpp - sdp_primal_dual_test.cpp - sgd_test.cpp - sgdr_test.cpp - snapshot_ensembles.cpp - serialization.hpp - serialization.cpp - serialization_test.cpp - sfinae_test.cpp - smorms3_test.cpp - softmax_regression_test.cpp - sort_policy_test.cpp - sparse_autoencoder_test.cpp - sparse_coding_test.cpp - spill_tree_test.cpp - split_data_test.cpp - svd_batch_test.cpp - svd_incremental_test.cpp - termination_policy_test.cpp - tree_test.cpp - tree_traits_test.cpp - union_find_test.cpp - svd_batch_test.cpp - svd_incremental_test.cpp - nystroem_method_test.cpp - armadillo_svd_test.cpp - ub_tree_test.cpp - vantage_point_tree_test.cpp - prefixedoutstream_test.cpp - timer_test.cpp + #pca_test.cpp + #perceptron_test.cpp + #q_learning_test.cpp + #qdafn_test.cpp + #quic_svd_test.cpp + #radical_test.cpp + #randomized_svd_test.cpp + #range_search_test.cpp + #recurrent_network_test.cpp + #rectangle_tree_test.cpp + #regularized_svd_test.cpp + #rl_components_test.cpp + #rmsprop_test.cpp + #sa_test.cpp + #sdp_primal_dual_test.cpp + #sgd_test.cpp + #sgdr_test.cpp + #snapshot_ensembles.cpp + #serialization.hpp + #serialization.cpp + #serialization_test.cpp + #sfinae_test.cpp + #smorms3_test.cpp + #softmax_regression_test.cpp + #sort_policy_test.cpp + #sparse_autoencoder_test.cpp + #sparse_coding_test.cpp + #spill_tree_test.cpp + #split_data_test.cpp + #svd_batch_test.cpp + #svd_incremental_test.cpp + #termination_policy_test.cpp + #tree_test.cpp + #tree_traits_test.cpp + #union_find_test.cpp + #svd_batch_test.cpp + #svd_incremental_test.cpp + #nystroem_method_test.cpp + #armadillo_svd_test.cpp + #ub_tree_test.cpp + #vantage_point_tree_test.cpp + #prefixedoutstream_test.cpp + #timer_test.cpp ) # Link dependencies of test executable. target_link_libraries(mlpack_test diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index 0654f20bfc..693207063b 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) { SparseTestFunction f; - ConstantStep decayPolicy(0.1); + ConstantStep decayPolicy(0.5); ParallelSGD s(f, 10000, 1, 1e-5, decayPolicy); @@ -48,10 +48,10 @@ BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) BOOST_REQUIRE_CLOSE(result, 123.75, 0.01); // The co-ordinates should be the vertices of the parabolae. - BOOST_REQUIRE_CLOSE(coordinates[0], 2, 0.1); - BOOST_REQUIRE_CLOSE(coordinates[1], 1, 0.1); - BOOST_REQUIRE_CLOSE(coordinates[2], 1.5, 0.1); - BOOST_REQUIRE_CLOSE(coordinates[3], 4, 0.1); + BOOST_REQUIRE_CLOSE(coordinates[0], 2, 0.02); + BOOST_REQUIRE_CLOSE(coordinates[1], 1, 0.02); + BOOST_REQUIRE_CLOSE(coordinates[2], 1.5, 0.02); + BOOST_REQUIRE_CLOSE(coordinates[3], 4, 0.02); } /** @@ -61,15 +61,15 @@ BOOST_AUTO_TEST_CASE(ExponentialBackoffDecayTest) { ExponentialBackoff decayPolicy(100, 100, 0.9); - // At the first iteration, the decay should be unchanged + // At the first iteration, stepsize should be unchanged BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(1), 100); - // At the 99th iteration + // At the 99th iteration, stepsize should be unchanged BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(99), 100); - // At the 100th iteration, decay should be changed + // At the 100th iteration, stepsize should be changed BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(100), 90); - // At the 210th iteration, decay should be unchanged + // At the 210th iteration, stepsize should be unchanged BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(210), 90); - // At the 211th iteration, decay should be changed + // At the 211th iteration, stepsize should be changed BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(211), 81); } From 7d17a8dc8db768545ebe4ac5fd26237373c737af Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 26 Jun 2017 20:10:40 +0530 Subject: [PATCH 14/41] Fix for Visual Studio brace initialization bug --- src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp | 5 +++++ .../core/optimizers/parallel_sgd/sparse_test_function.hpp | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 340c72144c..1602737b20 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -47,10 +47,15 @@ namespace optimization { * would refer to the index of the datapoint(or training example). * The data is distributed uniformly among the threads made available to the * program by the OpenMP runtime. + * * The class is expected to implement a Components function, which takes in the * index of a datapoint and returns a list of component indices(of the decision * variable) for which the decision variable needs to be updated. * + * If the function is not found, the gradient update will iterate through the + * entire gradient and update each component of the decision variable if the + * gradient is non-zero in that component. + * * @tparam SparseFunctionType Sparse, Decomposable objective function type to be * minimized. * @tparam DecayPolicyType Step size update policy used by parallel SGD diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp index 2bf7485d20..f65d343c87 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp @@ -50,10 +50,10 @@ class SparseTestFunction // first order term is stored. //! The vector storing the intercepts - arma::vec intercepts = {20, 12, 15, 100}; + arma::vec intercepts = {{20, 12, 15, 100}}; //! The vector having coefficients of the first order term - arma::vec bi = {-4, -2, -3, -8}; + arma::vec bi = {{-4, -2, -3, -8}}; }; } // namespace test From 75adb197801b57823a49ae6f708f04a9a81b8358 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Tue, 27 Jun 2017 00:30:45 +0530 Subject: [PATCH 15/41] Fix Visual Studio brace init issue and SimpleParallelSGDTest --- .../decay_policies/exponential_backoff.hpp | 2 +- .../parallel_sgd/sparse_test_function.hpp | 8 +- .../sparse_test_function_impl.hpp | 7 + src/mlpack/tests/CMakeLists.txt | 218 +++++++++--------- src/mlpack/tests/parallel_sgd_test.cpp | 11 +- 5 files changed, 130 insertions(+), 116 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index d797a01205..184241d47a 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -75,7 +75,7 @@ class ExponentialBackoff{ //! The initial stepsize. double step; - + //! The reduction factor, should be in range (0, 1). double beta; }; diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp index f65d343c87..9bce098ee4 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp @@ -26,8 +26,8 @@ namespace test { class SparseTestFunction { public: - //! Nothing to do for the constructor. - SparseTestFunction() { } + //! Set members in the default constructor. + SparseTestFunction(); //! Return 6 (the number of functions). size_t NumFunctions() const { return 4; } @@ -50,10 +50,10 @@ class SparseTestFunction // first order term is stored. //! The vector storing the intercepts - arma::vec intercepts = {{20, 12, 15, 100}}; + arma::vec intercepts; //! The vector having coefficients of the first order term - arma::vec bi = {{-4, -2, -3, -8}}; + arma::vec bi; }; } // namespace test diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp index 55144c55c9..feac875191 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp @@ -19,6 +19,13 @@ namespace mlpack { namespace optimization { namespace test { +//! The default constructor sets the members. +SparseTestFunction::SparseTestFunction() +{ + intercepts = arma::vec("20 12 15 100"); + bi = arma::vec("-4 -2 -3 -8"); +} + //! Evaluate a function. double SparseTestFunction::Evaluate( const arma::mat& coordinates, const size_t i) const diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 4df1839c1c..4a9096c5bd 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -1,116 +1,116 @@ # mlpack test executable. add_executable(mlpack_test - #activation_functions_test.cpp - #adaboost_test.cpp - #adam_test.cpp - #ada_delta_test.cpp - #ada_grad_test.cpp - #akfn_test.cpp - #aknn_test.cpp - #ann_layer_test.cpp - #arma_extend_test.cpp - #armadillo_svd_test.cpp - #aug_lagrangian_test.cpp - #binarize_test.cpp - #block_krylov_svd_test.cpp - #cf_test.cpp - #cli_test.cpp - #convolution_test.cpp - #convolutional_network_test.cpp - #cosine_tree_test.cpp - #cv_test.cpp - #dbscan_test.cpp - #decision_stump_test.cpp - #decision_tree_test.cpp - #det_test.cpp - #distribution_test.cpp - #drusilla_select_test.cpp - #emst_test.cpp - #fastmks_test.cpp - #feedforward_network_test.cpp - #gmm_test.cpp - #gradient_descent_test.cpp - #hmm_test.cpp - #hoeffding_tree_test.cpp - #hyperplane_test.cpp - #imputation_test.cpp - #ind2sub_test.cpp - #init_rules_test.cpp - #kernel_test.cpp - #kernel_pca_test.cpp - #kernel_traits_test.cpp - #kfn_test.cpp - #kmeans_test.cpp - #knn_test.cpp - #krann_search_test.cpp - #ksinit_test.cpp - #lars_test.cpp - #lbfgs_test.cpp - #lin_alg_test.cpp - #linear_regression_test.cpp - #load_save_test.cpp - #local_coordinate_coding_test.cpp - #log_test.cpp - #logistic_regression_test.cpp - #lrsdp_test.cpp - #lsh_test.cpp - #math_test.cpp - #matrix_completion_test.cpp - #maximal_inputs_test.cpp - #mean_shift_test.cpp - #metric_test.cpp - #minibatch_sgd_test.cpp + activation_functions_test.cpp + adaboost_test.cpp + adam_test.cpp + ada_delta_test.cpp + ada_grad_test.cpp + akfn_test.cpp + aknn_test.cpp + ann_layer_test.cpp + arma_extend_test.cpp + armadillo_svd_test.cpp + aug_lagrangian_test.cpp + binarize_test.cpp + block_krylov_svd_test.cpp + cf_test.cpp + cli_test.cpp + convolution_test.cpp + convolutional_network_test.cpp + cosine_tree_test.cpp + cv_test.cpp + dbscan_test.cpp + decision_stump_test.cpp + decision_tree_test.cpp + det_test.cpp + distribution_test.cpp + drusilla_select_test.cpp + emst_test.cpp + fastmks_test.cpp + feedforward_network_test.cpp + gmm_test.cpp + gradient_descent_test.cpp + hmm_test.cpp + hoeffding_tree_test.cpp + hyperplane_test.cpp + imputation_test.cpp + ind2sub_test.cpp + init_rules_test.cpp + kernel_test.cpp + kernel_pca_test.cpp + kernel_traits_test.cpp + kfn_test.cpp + kmeans_test.cpp + knn_test.cpp + krann_search_test.cpp + ksinit_test.cpp + lars_test.cpp + lbfgs_test.cpp + lin_alg_test.cpp + linear_regression_test.cpp + load_save_test.cpp + local_coordinate_coding_test.cpp + log_test.cpp + logistic_regression_test.cpp + lrsdp_test.cpp + lsh_test.cpp + math_test.cpp + matrix_completion_test.cpp + maximal_inputs_test.cpp + mean_shift_test.cpp + metric_test.cpp + minibatch_sgd_test.cpp mlpack_test.cpp - #momentum_sgd_test.cpp - #nbc_test.cpp - #nca_test.cpp - #nmf_test.cpp - #nystroem_method_test.cpp - #octree_test.cpp + momentum_sgd_test.cpp + nbc_test.cpp + nca_test.cpp + nmf_test.cpp + nystroem_method_test.cpp + octree_test.cpp parallel_sgd_test.cpp - #pca_test.cpp - #perceptron_test.cpp - #q_learning_test.cpp - #qdafn_test.cpp - #quic_svd_test.cpp - #radical_test.cpp - #randomized_svd_test.cpp - #range_search_test.cpp - #recurrent_network_test.cpp - #rectangle_tree_test.cpp - #regularized_svd_test.cpp - #rl_components_test.cpp - #rmsprop_test.cpp - #sa_test.cpp - #sdp_primal_dual_test.cpp - #sgd_test.cpp - #sgdr_test.cpp - #snapshot_ensembles.cpp - #serialization.hpp - #serialization.cpp - #serialization_test.cpp - #sfinae_test.cpp - #smorms3_test.cpp - #softmax_regression_test.cpp - #sort_policy_test.cpp - #sparse_autoencoder_test.cpp - #sparse_coding_test.cpp - #spill_tree_test.cpp - #split_data_test.cpp - #svd_batch_test.cpp - #svd_incremental_test.cpp - #termination_policy_test.cpp - #tree_test.cpp - #tree_traits_test.cpp - #union_find_test.cpp - #svd_batch_test.cpp - #svd_incremental_test.cpp - #nystroem_method_test.cpp - #armadillo_svd_test.cpp - #ub_tree_test.cpp - #vantage_point_tree_test.cpp - #prefixedoutstream_test.cpp - #timer_test.cpp + pca_test.cpp + perceptron_test.cpp + q_learning_test.cpp + qdafn_test.cpp + quic_svd_test.cpp + radical_test.cpp + randomized_svd_test.cpp + range_search_test.cpp + recurrent_network_test.cpp + rectangle_tree_test.cpp + regularized_svd_test.cpp + rl_components_test.cpp + rmsprop_test.cpp + sa_test.cpp + sdp_primal_dual_test.cpp + sgd_test.cpp + sgdr_test.cpp + snapshot_ensembles.cpp + serialization.hpp + serialization.cpp + serialization_test.cpp + sfinae_test.cpp + smorms3_test.cpp + softmax_regression_test.cpp + sort_policy_test.cpp + sparse_autoencoder_test.cpp + sparse_coding_test.cpp + spill_tree_test.cpp + split_data_test.cpp + svd_batch_test.cpp + svd_incremental_test.cpp + termination_policy_test.cpp + tree_test.cpp + tree_traits_test.cpp + union_find_test.cpp + svd_batch_test.cpp + svd_incremental_test.cpp + nystroem_method_test.cpp + armadillo_svd_test.cpp + ub_tree_test.cpp + vantage_point_tree_test.cpp + prefixedoutstream_test.cpp + timer_test.cpp ) # Link dependencies of test executable. target_link_libraries(mlpack_test diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index 693207063b..d81eb9070a 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -35,9 +35,16 @@ BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) { SparseTestFunction f; - ConstantStep decayPolicy(0.5); + ConstantStep decayPolicy(0.4); - ParallelSGD s(f, 10000, 1, 1e-5, + // The batch size for this test should be chosen according to the threads + // available on the system. If the update does not touch each datapoint, the + // test will fail. + + size_t threadsAvailable = omp_get_max_threads(); + size_t batchSize = std::ceil((float) f.NumFunctions() / threadsAvailable); + + ParallelSGD s(f, 10000, batchSize, 1e-5, decayPolicy); arma::mat coordinates = f.GetInitialPoint(); From a116ab2b7ac91b02cf60b04f28493ac61090035f Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Wed, 28 Jun 2017 18:27:25 +0530 Subject: [PATCH 16/41] Changes to SparseFunctionType interface --- .../optimizers/parallel_sgd/parallel_sgd.hpp | 16 ++++++--------- .../parallel_sgd/parallel_sgd_impl.hpp | 20 +++++++++++-------- .../parallel_sgd/sparse_svm_function.hpp | 15 ++++++-------- .../parallel_sgd/sparse_svm_function_impl.hpp | 19 ++++-------------- .../parallel_sgd/sparse_test_function.hpp | 5 +---- .../sparse_test_function_impl.hpp | 9 ++------- 6 files changed, 31 insertions(+), 53 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 1602737b20..9b8179c2b6 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -39,8 +39,7 @@ namespace optimization { * double Evaluate(const arma::mat& coordinates, const size_t i); * void Gradient(const arma::mat& coordinates, * const size_t i, - * arma::mat& gradient); - * arma::Col Components(size_t id); + * arma::sp_mat& gradient); * * In these functions the parameter id refers to which individual function (or * gradient) is being evaluated. In case of a data-dependent function, the id @@ -48,13 +47,10 @@ namespace optimization { * The data is distributed uniformly among the threads made available to the * program by the OpenMP runtime. * - * The class is expected to implement a Components function, which takes in the - * index of a datapoint and returns a list of component indices(of the decision - * variable) for which the decision variable needs to be updated. - * - * If the function is not found, the gradient update will iterate through the - * entire gradient and update each component of the decision variable if the - * gradient is non-zero in that component. + * The Gradient function interface is slightly changed from the + * DecomposableFunctionType interface, it takes in a sparse matrix as the + * out-param for the gradient. As ParallelSGD is only expected to be relevant in + * situations where the computed gradient is sparse. * * @tparam SparseFunctionType Sparse, Decomposable objective function type to be * minimized. @@ -142,7 +138,7 @@ class ParallelSGD /** * Generate the indices to be visited by each thread before iteration. * Generates a randomly shuffled vector of datapoint indices (range 0 to - * function.NumFunctions()). + * function.NumFunctions() - 1). * * @param visitationOrder Out param with the indices of the datapoints for the * current iteration. diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 9b0b17735d..ee441e04da 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -51,22 +51,26 @@ double ParallelSGD::Optimize( #pragma omp parallel { - // Each processor gets a subset of the instances - // Each subset is of size batchSize + // Each processor gets a subset of the instances. + // Each subset is of size batchSize. arma::Col instances = ThreadShare(omp_get_thread_num(), visitationOrder); for (size_t j = 0; j < instances.n_elem; ++j) { - // Each instance affects only some components of the decision variable - arma::Col components = function.Components(instances[j]); - // Evaluate the gradient - arma::vec gradient; + // Each instance affects only some components of the decision variable. + // So the gradient is sparse. + arma::sp_mat gradient; + + // Evaluate the sparse gradient. function.Gradient(iterate, instances[j], gradient); - for (size_t k = 0; k < components.n_elem; ++k) + // Update the decision variable with non-zero components of the + // gradient. + for (auto cur = gradient.begin_col(0); cur != gradient.end_col(0); + ++cur) { #pragma omp atomic - iterate[components[k]] -= stepSize * gradient[components[k]]; + iterate[cur.row()] -= stepSize * gradient[cur.row()]; } } } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp index 00f66566b4..3fe2288c34 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -17,24 +17,21 @@ class SparseSVMLossFunction{ public: //! Nothing to do for the default constructor. - SparseSVMLossFunction(); + SparseSVMLossFunction() = default; //! Member initialization constructor. - SparseSVMLossFunction(arma::SpMat& dataset, arma::vec& labels); + SparseSVMLossFunction(arma::sp_mat& dataset, arma::vec& labels); //! Evaluate a function. double Evaluate(const arma::vec& weights, size_t id); //! Evaluate the gradient of a function. - void Gradient(const arma::vec& weights, size_t id, arma::mat& gradient); - - //! Get the list of non-zero components of the gradient of a function. - arma::Col Components(size_t id); + void Gradient(const arma::vec& weights, size_t id, arma::sp_mat& gradient); //! Get the dataset. - const arma::SpMat& Dataset() const { return dataset; } + const arma::sp_mat& Dataset() const { return dataset; } //! Modify the dataset. - arma::SpMat& Dataset() { return dataset; } + arma::sp_mat& Dataset() { return dataset; } //! Get the labels. const arma::vec& Labels() const { return labels; } @@ -46,7 +43,7 @@ class SparseSVMLossFunction{ private: //! The datapoints for training. - arma::SpMat dataset; + arma::sp_mat dataset; //! The labels, y_i. arma::vec labels; diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp index 95e9de6aa5..619269ce03 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -17,7 +17,7 @@ #include "sparse_svm_function.hpp" SparseSVMLossFunction::SparseSVMLossFunction( - arma::SpMat& dataset, arma::vec& labels) : + arma::sp_mat& dataset, arma::vec& labels) : dataset(dataset), labels(labels) { /* Nothing to do */ } @@ -27,21 +27,11 @@ double SparseSVMLossFunction::Evaluate(const arma::vec& weights, size_t id) } void SparseSVMLossFunction::Gradient( - const arma::vec& weights, size_t id, arma::mat& gradient) + const arma::vec& weights, size_t id, arma::sp_mat& gradient) { double dot = 1 - labels(id) * arma::dot(weights, dataset.col(id)); - gradient = (dot < 0) ? arma::vec(weights.n_elem, arma::fill::zeros) : - (-1 * arma::vec(dataset.col(id) * labels(id))); -} - -arma::Col SparseSVMLossFunction::Components(size_t id) -{ - std::vector nonZeroComponents; - for (auto cur = dataset.begin_col(id); cur != dataset.end_col(id); ++cur) - { - nonZeroComponents.push_back(cur.row()); - } - return arma::Col(nonZeroComponents); + gradient = (dot < 0) ? arma::sp_mat(weights.n_rows, 1) : + (-1 * arma::sp_mat(dataset.col(id) * labels(id))); } size_t SparseSVMLossFunction::NumFunctions() @@ -49,5 +39,4 @@ size_t SparseSVMLossFunction::NumFunctions() return dataset.n_cols; } - #endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp index 9bce098ee4..2f48cd466d 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp @@ -35,16 +35,13 @@ class SparseTestFunction //! Get the starting point. arma::mat GetInitialPoint() const { return arma::mat("0; 0; 0; 0;"); } - //! Get the list of non-zero components of the gradient of a function. - arma::Col Components(size_t id); - //! Evaluate a function. double Evaluate(const arma::mat& coordinates, const size_t i) const; //! Evaluate the gradient of a function. void Gradient(const arma::mat& coordinates, const size_t i, - arma::mat& gradient) const; + arma::sp_mat& gradient) const; private: // Each quadratic polynomial is monic. The intercept and coefficient of the // first order term is stored. diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp index feac875191..e7c4568a08 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function_impl.hpp @@ -37,17 +37,12 @@ double SparseTestFunction::Evaluate( //! Evaluate the gradient of a function. void SparseTestFunction::Gradient(const arma::mat& coordinates, const size_t i, - arma::mat& gradient) const + arma::sp_mat& gradient) const { - gradient = arma::vec(coordinates.n_rows, 1, arma::fill::zeros); + gradient = arma::sp_mat(coordinates.n_rows, 1); gradient[i] = 2 * coordinates[i] + bi[i]; } -arma::Col SparseTestFunction::Components(size_t id) -{ - return arma::Col({ id }); -} - } // namespace test } // namespace optimization } // namespace mlpack From 523251153e6b0906f6819ae3e3640826f4d6eb7c Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sun, 2 Jul 2017 03:09:13 +0530 Subject: [PATCH 17/41] Initial implementation of sparse Matrix Completion loss function --- .../parallel_sgd/parallel_sgd_impl.hpp | 12 ++- .../parallel_sgd/sparse_mc_function.hpp | 80 +++++++++++++++++ .../parallel_sgd/sparse_mc_function_impl.hpp | 85 +++++++++++++++++++ .../parallel_sgd/sparse_svm_function.hpp | 6 +- .../parallel_sgd/sparse_svm_function_impl.hpp | 6 +- 5 files changed, 179 insertions(+), 10 deletions(-) create mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp create mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index ee441e04da..5e1c462b5b 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -66,11 +66,15 @@ double ParallelSGD::Optimize( // Update the decision variable with non-zero components of the // gradient. - for (auto cur = gradient.begin_col(0); cur != gradient.end_col(0); - ++cur) + for(size_t i = 0; i < gradient.n_cols; ++i) { - #pragma omp atomic - iterate[cur.row()] -= stepSize * gradient[cur.row()]; + // Iterate over the non-zero elements. + for (auto cur = gradient.begin_col(i); cur != gradient.end_col(i); + ++cur) + { + #pragma omp atomic + iterate(cur.row(), i) -= stepSize * gradient(cur.row(), i); + } } } } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp new file mode 100644 index 0000000000..22bff5bfad --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp @@ -0,0 +1,80 @@ +/** + * @file sparse_mc_function.hpp + * @author Shikhar Bhardwaj + * + * Implementation of the hinge loss function for training a sparse SVM with the + * parallel SGD algorithm + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_HPP +#include + +class SparseMCLossFunction{ + public: + //! Nothing to do for the default constructor. + SparseMCLossFunction() = default; + + //! Member initialization constructor. + SparseMCLossFunction(arma::uvec& rows, arma::uvec& cols, arma::vec& ratings, + size_t rank); + + //! Special Initialization constructor. + SparseMCLossFunction(arma::sp_mat& dataset, size_t rank); + + //! Evaluate a function. + double Evaluate(const arma::mat& weights, size_t id); + + //! Evaluate the gradient of a function. + void Gradient(const arma::mat& weights, size_t id, arma::sp_mat& gradient); + + //! Get the height of the sparse matrix. + size_t NumRows() const { return numRows; } + //! Modify the height of the sparse matrix. + size_t& NumRows() { return numRows; } + + //! Get the width of the sparse matrix. + size_t NumCols() const { return numCols; } + //! Modify the width of the sparse matrix. + size_t& NumCols() { return numCols; } + + //! Get the rank. + size_t Rank() const { return rank; } + //! Modify the rank. + size_t& Rank() { return rank; } + + //! Return the number of functions. + size_t NumFunctions(); + + private: + void CalculateStatistics(); + + //! The training data. + arma::uvec rows; + arma::uvec cols; + arma::vec ratings; + + //! The statistics. + arma::uvec colCnt; + arma::uvec rowCnt; + //! Mean rating. + double meanRating; + + //! The height of the sparse matrix + size_t numRows; + + //! The width of the sparse matrix + size_t numCols; + + //! The width of the first factor. + size_t rank; +}; + +// Include implementation +#include "sparse_mc_function_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp new file mode 100644 index 0000000000..e622d4109e --- /dev/null +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp @@ -0,0 +1,85 @@ +/** + * @file sparse_mc_function_impl.hpp + * @author Shikhar Bhardwaj + * + * Implementation of the sparse matrix factorization example loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_IMPL_HPP + +// In case it hasn't been included yet. +#include "sparse_mc_function.hpp" + +SparseMCLossFunction::SparseMCLossFunction(arma::uvec &rows, arma::uvec &cols, + arma::vec &ratings, size_t rank) : + rows(rows), cols(cols), ratings(ratings), rank(rank) +{ /* Nothing to do */ } + +SparseMCLossFunction::SparseMCLossFunction(arma::sp_mat &dataset, + size_t rank) : rank(rank) +{ + // Extract the relevant data from the sparse matrix representation. + std::vector instance_rows, instance_cols, instance_ratings; + for(size_t i = 0; i < dataset.n_cols; ++i) + { + for(auto cur = dataset.begin_col(i); cur != dataset.end_col(i); ++cur) + { + instance_cols.push_back(i); + instance_rows.push_back(cur.row()); + instance_ratings.push_back(*cur); + } + } + // Store the data in the object state. + rows = arma::uvec(instance_rows); + cols = arma::uvec(instance_cols); + ratings = arma::vec(instance_ratings); +} + +void SparseMCLossFunction::CalculateStatistics() +{ + // Take one pass over the data to aggregate statistics. + size_t n_cols = arma::max(cols); + size_t n_rows = arma::max(rows); + // Initialize the statistics aggregate structure. + colCnt = arma::uvec(n_cols, arma::fill::zeros); + rowCnt = arma::uvec(n_rows, arma::fill::zeros); + // Go through the data and calculate the required frequency. + for(size_t i = 0; i < rows.n_elem; ++i) + { + rowCnt[rows[i]]++; + colCnt[cols[i]]++; + } + meanRating = arma::mean(ratings); +} + +double SparseMCLossFunction::Evaluate(const arma::mat &weights, size_t id) +{ + // The decision variable is expected to be stored as follows. + // The first numRows columns have the first factor matrix, the next numCols + // columns have the second factor matrix. The decision variable matrix is + // thus of size (numRows + numCols) x rank. + float error = arma::dot(weights.col(rows(id)), weights.col(numRows + id)) + + meanRating - ratings(id); + return error * error; +} + +void SparseMCLossFunction::Gradient(const arma::mat &weights, size_t id, + arma::sp_mat &gradient) +{ + gradient = arma::sp_mat(numRows + numCols, rank); + // We only need to alter the relevant row and column in the decision variable. + double error = arma::Dot(weights.col(rows(id)), weights.col(numRows + id)) + + meanRating - ratings(id); +} + +size_t SparseMCLossFunction::NumFunctions() +{ + return rows.n_elem; +} + +#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp index 3fe2288c34..76f0582314 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp @@ -1,5 +1,5 @@ /** - * @file parallel_sgd_impl.hpp + * @file sparse_svm_function.hpp * @author Shikhar Bhardwaj * * Implementation of the hinge loss function for training a sparse SVM with the @@ -23,10 +23,10 @@ class SparseSVMLossFunction{ SparseSVMLossFunction(arma::sp_mat& dataset, arma::vec& labels); //! Evaluate a function. - double Evaluate(const arma::vec& weights, size_t id); + double Evaluate(const arma::mat& weights, size_t id); //! Evaluate the gradient of a function. - void Gradient(const arma::vec& weights, size_t id, arma::sp_mat& gradient); + void Gradient(const arma::mat& weights, size_t id, arma::sp_mat& gradient); //! Get the dataset. const arma::sp_mat& Dataset() const { return dataset; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp index 619269ce03..386d2937c9 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp @@ -1,5 +1,5 @@ /** - * @file parallel_sgd_impl.hpp + * @file sparse_svm_function_impl.hpp * @author Shikhar Bhardwaj * * Implementation of the hinge loss function for training a sparse SVM with the @@ -21,13 +21,13 @@ SparseSVMLossFunction::SparseSVMLossFunction( dataset(dataset), labels(labels) { /* Nothing to do */ } -double SparseSVMLossFunction::Evaluate(const arma::vec& weights, size_t id) +double SparseSVMLossFunction::Evaluate(const arma::mat& weights, size_t id) { return std::max(0.0, 1 - labels(id) * arma::dot(dataset.col(id), weights)); } void SparseSVMLossFunction::Gradient( - const arma::vec& weights, size_t id, arma::sp_mat& gradient) + const arma::mat& weights, size_t id, arma::sp_mat& gradient) { double dot = 1 - labels(id) * arma::dot(weights, dataset.col(id)); gradient = (dot < 0) ? arma::sp_mat(weights.n_rows, 1) : From 810cc149ea11ed39f925f4040ae3115c23198f2d Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sun, 2 Jul 2017 17:37:13 +0530 Subject: [PATCH 18/41] Implementation of sparse Matrix Completion loss function complete --- .../parallel_sgd/parallel_sgd_impl.hpp | 2 +- .../parallel_sgd/sparse_mc_function.hpp | 14 +++- .../parallel_sgd/sparse_mc_function_impl.hpp | 79 +++++++++++++------ 3 files changed, 68 insertions(+), 27 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 5e1c462b5b..02e7410300 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -66,7 +66,7 @@ double ParallelSGD::Optimize( // Update the decision variable with non-zero components of the // gradient. - for(size_t i = 0; i < gradient.n_cols; ++i) + for (size_t i = 0; i < gradient.n_cols; ++i) { // Iterate over the non-zero elements. for (auto cur = gradient.begin_col(i); cur != gradient.end_col(i); diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp index 22bff5bfad..87c96dcd7d 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp @@ -20,11 +20,11 @@ class SparseMCLossFunction{ SparseMCLossFunction() = default; //! Member initialization constructor. - SparseMCLossFunction(arma::uvec& rows, arma::uvec& cols, arma::vec& ratings, - size_t rank); + SparseMCLossFunction(const arma::uvec& rows, const arma::uvec& cols, + const arma::vec& ratings, double mu, size_t rank); //! Special Initialization constructor. - SparseMCLossFunction(arma::sp_mat& dataset, size_t rank); + SparseMCLossFunction(const arma::sp_mat& dataset, double mu, size_t rank); //! Evaluate a function. double Evaluate(const arma::mat& weights, size_t id); @@ -42,6 +42,11 @@ class SparseMCLossFunction{ //! Modify the width of the sparse matrix. size_t& NumCols() { return numCols; } + //! Get the regularization parameter. + double Mu() const { return mu; } + //! Modify the regularization parameter. + double& Mu() { return mu; } + //! Get the rank. size_t Rank() const { return rank; } //! Modify the rank. @@ -64,6 +69,9 @@ class SparseMCLossFunction{ //! Mean rating. double meanRating; + //! The regularization parameter. + double mu; + //! The height of the sparse matrix size_t numRows; diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp index e622d4109e..f63b5c835e 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp @@ -15,19 +15,25 @@ // In case it hasn't been included yet. #include "sparse_mc_function.hpp" -SparseMCLossFunction::SparseMCLossFunction(arma::uvec &rows, arma::uvec &cols, - arma::vec &ratings, size_t rank) : - rows(rows), cols(cols), ratings(ratings), rank(rank) -{ /* Nothing to do */ } +SparseMCLossFunction::SparseMCLossFunction(const arma::uvec& rows, + const arma::uvec& cols, + const arma::vec& ratings, + double mu, + size_t rank) : + rows(rows), cols(cols), ratings(ratings), mu(mu), rank(rank) +{ + CalculateStatistics(); +} -SparseMCLossFunction::SparseMCLossFunction(arma::sp_mat &dataset, - size_t rank) : rank(rank) +SparseMCLossFunction::SparseMCLossFunction(const arma::sp_mat& dataset, + double mu, + size_t rank) : mu(mu), rank(rank) { // Extract the relevant data from the sparse matrix representation. std::vector instance_rows, instance_cols, instance_ratings; - for(size_t i = 0; i < dataset.n_cols; ++i) + for (size_t i = 0; i < dataset.n_cols; ++i) { - for(auto cur = dataset.begin_col(i); cur != dataset.end_col(i); ++cur) + for (auto cur = dataset.begin_col(i); cur != dataset.end_col(i); ++cur) { instance_cols.push_back(i); instance_rows.push_back(cur.row()); @@ -38,43 +44,70 @@ SparseMCLossFunction::SparseMCLossFunction(arma::sp_mat &dataset, rows = arma::uvec(instance_rows); cols = arma::uvec(instance_cols); ratings = arma::vec(instance_ratings); + CalculateStatistics(); } void SparseMCLossFunction::CalculateStatistics() { // Take one pass over the data to aggregate statistics. - size_t n_cols = arma::max(cols); - size_t n_rows = arma::max(rows); + numCols = arma::max(cols) + 1; + numRows = arma::max(rows) + 1; // Initialize the statistics aggregate structure. - colCnt = arma::uvec(n_cols, arma::fill::zeros); - rowCnt = arma::uvec(n_rows, arma::fill::zeros); - // Go through the data and calculate the required frequency. - for(size_t i = 0; i < rows.n_elem; ++i) + colCnt = arma::uvec(numCols, arma::fill::zeros); + rowCnt = arma::uvec(numRows, arma::fill::zeros); + // Go through the data and calculate the required frequencies. + for (size_t i = 0; i < rows.n_elem; ++i) { - rowCnt[rows[i]]++; - colCnt[cols[i]]++; + rowCnt(rows(i))++; + colCnt(cols(i))++; } meanRating = arma::mean(ratings); } -double SparseMCLossFunction::Evaluate(const arma::mat &weights, size_t id) +double SparseMCLossFunction::Evaluate(const arma::mat& weights, size_t id) { // The decision variable is expected to be stored as follows. // The first numRows columns have the first factor matrix, the next numCols // columns have the second factor matrix. The decision variable matrix is // thus of size (numRows + numCols) x rank. - float error = arma::dot(weights.col(rows(id)), weights.col(numRows + id)) + + + size_t colId = numRows + cols(id); + size_t rowId = rows(id); + + float error = arma::dot(weights.col(rowId), weights.col(colId)) + meanRating - ratings(id); - return error * error; + float loss = error * error; + if (rowCnt(rows(id)) > 1) + loss += mu * arma::norm(weights.col(rowId)) / (2 * (rowCnt(rows(id)) - 1)); + if (colCnt(cols(id)) > 1) + loss += mu * arma::norm(weights.col(colId)) / (2 * (colCnt(cols(id)) - 1)); + return loss; } -void SparseMCLossFunction::Gradient(const arma::mat &weights, size_t id, - arma::sp_mat &gradient) +void SparseMCLossFunction::Gradient(const arma::mat& weights, size_t id, + arma::sp_mat& gradient) { + // Index of the column corresponding to the row and column of the current + // example in the decision variable. + size_t colId = numRows + cols(id); + size_t rowId = rows(id); + gradient = arma::sp_mat(numRows + numCols, rank); - // We only need to alter the relevant row and column in the decision variable. - double error = arma::Dot(weights.col(rows(id)), weights.col(numRows + id)) + + double error = arma::dot(weights.col(rowId), weights.col(colId)) + meanRating - ratings(id); + + // Calculate gradient for the first factor. + if (rowCnt(rows(id)) > 1) + gradient.col(rowId) = (mu / (rowCnt(rows(id)) - 1)) * weights.col(rowId); + + gradient.col(rowId) += error * weights.col(colId); + + // Calculate gradient for the second factor. + if (colCnt(cols(id)) > 1) + gradient.col(colId) = (mu / (colCnt(cols(id)) - 1)) * weights.col(colId); + + gradient.col(colId) += error * weights.col(rowId); } size_t SparseMCLossFunction::NumFunctions() From f7be9e7d6c81e1342cabfcf822e9f5b0d57da539 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sun, 2 Jul 2017 21:46:03 +0530 Subject: [PATCH 19/41] Improve comments in matrix completion example --- .../optimizers/parallel_sgd/CMakeLists.txt | 2 ++ .../parallel_sgd/sparse_mc_function.hpp | 18 ++++++++++++++++-- .../parallel_sgd/sparse_mc_function_impl.hpp | 11 +++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt index 07c5dc517a..ef9ea07c48 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt +++ b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt @@ -3,6 +3,8 @@ set(SOURCES parallel_sgd_impl.hpp sparse_svm_function.hpp sparse_svm_function_impl.hpp + sparse_mc_function.hpp + sparse_mc_function_impl.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp index 87c96dcd7d..c5b8103984 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp @@ -14,6 +14,11 @@ #define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_HPP #include +/** + * An implementation of the matrix completion example from HOGWILD!, based on + * empirical risk minimization in a sparse setting. + */ + class SparseMCLossFunction{ public: //! Nothing to do for the default constructor. @@ -56,16 +61,25 @@ class SparseMCLossFunction{ size_t NumFunctions(); private: + //! Calculate the frequency tables and mean rating before calling Evaluate + //! or Gradient. void CalculateStatistics(); - //! The training data. + //! The row index of the datapoints. arma::uvec rows; + + //! The column index of the datapoints. arma::uvec cols; + + //! The rating of the datapoints. arma::vec ratings; - //! The statistics. + //! The frequency of the columns. arma::uvec colCnt; + + //! The frequency of the rows. arma::uvec rowCnt; + //! Mean rating. double meanRating; diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp index f63b5c835e..8e2aa65a0f 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp @@ -30,7 +30,8 @@ SparseMCLossFunction::SparseMCLossFunction(const arma::sp_mat& dataset, size_t rank) : mu(mu), rank(rank) { // Extract the relevant data from the sparse matrix representation. - std::vector instance_rows, instance_cols, instance_ratings; + std::vector instance_rows, instance_cols; + std::vector instance_ratings; for (size_t i = 0; i < dataset.n_cols; ++i) { for (auto cur = dataset.begin_col(i); cur != dataset.end_col(i); ++cur) @@ -52,9 +53,10 @@ void SparseMCLossFunction::CalculateStatistics() // Take one pass over the data to aggregate statistics. numCols = arma::max(cols) + 1; numRows = arma::max(rows) + 1; - // Initialize the statistics aggregate structure. + colCnt = arma::uvec(numCols, arma::fill::zeros); rowCnt = arma::uvec(numRows, arma::fill::zeros); + // Go through the data and calculate the required frequencies. for (size_t i = 0; i < rows.n_elem; ++i) { @@ -77,10 +79,13 @@ double SparseMCLossFunction::Evaluate(const arma::mat& weights, size_t id) float error = arma::dot(weights.col(rowId), weights.col(colId)) + meanRating - ratings(id); float loss = error * error; + + // Add the regularisation term. if (rowCnt(rows(id)) > 1) loss += mu * arma::norm(weights.col(rowId)) / (2 * (rowCnt(rows(id)) - 1)); if (colCnt(cols(id)) > 1) loss += mu * arma::norm(weights.col(colId)) / (2 * (colCnt(cols(id)) - 1)); + return loss; } @@ -98,12 +103,14 @@ void SparseMCLossFunction::Gradient(const arma::mat& weights, size_t id, + meanRating - ratings(id); // Calculate gradient for the first factor. + // Add the regularisation term. if (rowCnt(rows(id)) > 1) gradient.col(rowId) = (mu / (rowCnt(rows(id)) - 1)) * weights.col(rowId); gradient.col(rowId) += error * weights.col(colId); // Calculate gradient for the second factor. + // Add the regularisation term. if (colCnt(cols(id)) > 1) gradient.col(colId) = (mu / (colCnt(cols(id)) - 1)) * weights.col(colId); From 3f1d96ab9770c99fc2d5344f52872ef7d19a387a Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sun, 2 Jul 2017 23:27:20 +0530 Subject: [PATCH 20/41] Update parallel SGD to match new Optimizer API --- .../optimizers/parallel_sgd/parallel_sgd.hpp | 42 ++++--------------- .../parallel_sgd/parallel_sgd_impl.hpp | 31 ++++++-------- .../parallel_sgd/sparse_mc_function.hpp | 2 +- src/mlpack/tests/parallel_sgd_test.cpp | 11 +++-- 4 files changed, 27 insertions(+), 59 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 9b8179c2b6..fd8cf4b736 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -13,7 +13,6 @@ #define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_HPP #include -#include namespace mlpack { namespace optimization { @@ -52,15 +51,10 @@ namespace optimization { * out-param for the gradient. As ParallelSGD is only expected to be relevant in * situations where the computed gradient is sparse. * - * @tparam SparseFunctionType Sparse, Decomposable objective function type to be - * minimized. * @tparam DecayPolicyType Step size update policy used by parallel SGD * to update the stepsize after each iteration. */ -template < - typename SparseFunctionType, - typename DecayPolicyType -> +template class ParallelSGD { public: @@ -69,15 +63,13 @@ class ParallelSGD * the given parameters. One iteration means one batch of datapoints processed * by each thread. * - * @param function Function to be optimized(minimized). * @param maxIterations Maximum number of iterations allowed. * @param batchSize Number of datapoints to be processed in one iteration by * each thread. * @param tolerance Maximum absolute tolerance to terminate the algorithm. * @param decayPolicy The step size update policy to use. */ - ParallelSGD(SparseFunctionType& function, - const size_t maxIterations, + ParallelSGD(const size_t maxIterations, const size_t batchSize, const double tolerance, const DecayPolicyType& decayPolicy); @@ -88,30 +80,14 @@ class ParallelSGD * algorithm, and the value of the loss function at the final point is * returned. * - * @param function Function to be opmtimized(minimized). + * @tparam SparseFunctionType Type of function to be optimized. + * @param function Function to be optimized(minimized). * @param iterate Starting point(will be modified). * @return Objective value at the final point. */ + template double Optimize(SparseFunctionType& function, arma::mat& iterate); - /** - * Optimize the given function using stochastic gradient descent. The given - * starting point will be modified to store the finishing point of the - * algorithm, and the final objective value is returned. - * - * @param iterate Starting point (will be modified). - * @return Objective value of the final point. - */ - double Optimize(arma::mat& iterate) - { - return Optimize(this->function, iterate); - } - - //! Get the instantiated function to be optimized. - const SparseFunctionType& Function() const { return function; } - //! Modify the instantiated function. - SparseFunctionType& Function() { return function; } - //! Get the maximum number of iterations (0 indicates no limits). size_t MaxIterations() const { return maxIterations; } //! Modify the maximum number of iterations (0 indicates no limits). @@ -142,8 +118,10 @@ class ParallelSGD * * @param visitationOrder Out param with the indices of the datapoints for the * current iteration. + * @param numFunctions The number of separable functions in the objective. */ - void GenerateVisitationOrder(arma::Col& visitationOrder); + void GenerateVisitationOrder(arma::Col& visitationOrder, + size_t numFunctions); /** * Get the share of datapoint indices to be updated by the thread with given @@ -157,10 +135,6 @@ class ParallelSGD arma::Col ThreadShare(size_t thread_id, const arma::Col& visitationOrder); - - //! The instantiated function. - SparseFunctionType& function; - //! The maximum number of allowed iterations. size_t maxIterations; diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 02e7410300..34ce9f773b 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -18,22 +18,21 @@ namespace mlpack { namespace optimization { -template -ParallelSGD::ParallelSGD( - SparseFunctionType& function, +template +ParallelSGD::ParallelSGD( const size_t maxIterations, const size_t batchSize, const double tolerance, const DecayPolicyType& decayPolicy) : - function(function), maxIterations(maxIterations), batchSize(batchSize), tolerance(tolerance), decayPolicy(decayPolicy) { /* Nothing to do. */ } -template -double ParallelSGD::Optimize( +template +template +double ParallelSGD::Optimize( SparseFunctionType& function, arma::mat& iterate) { @@ -47,7 +46,7 @@ double ParallelSGD::Optimize( double stepSize = decayPolicy.StepSize(i); arma::Col visitationOrder; - GenerateVisitationOrder(visitationOrder); + GenerateVisitationOrder(visitationOrder, function.NumFunctions()); #pragma omp parallel { @@ -100,21 +99,17 @@ double ParallelSGD::Optimize( return overallObjective; } -template -void ParallelSGD< - SparseFunctionType, - DecayPolicyType>::GenerateVisitationOrder( - arma::Col& visitationOrder) +template +void ParallelSGD::GenerateVisitationOrder( + arma::Col& visitationOrder, size_t numFunctions) { visitationOrder = arma::shuffle(arma::linspace>(0, - (function.NumFunctions() - 1), function.NumFunctions())); + (numFunctions - 1), numFunctions)); } -template -arma::Col ParallelSGD< - SparseFunctionType, - DecayPolicyType>::ThreadShare(size_t thread_id, - const arma::Col& visitationOrder) +template +arma::Col ParallelSGD::ThreadShare( + size_t thread_id, const arma::Col& visitationOrder) { if (thread_id * batchSize >= visitationOrder.n_elem) { diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp index c5b8103984..ec40baabb4 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp @@ -26,7 +26,7 @@ class SparseMCLossFunction{ //! Member initialization constructor. SparseMCLossFunction(const arma::uvec& rows, const arma::uvec& cols, - const arma::vec& ratings, double mu, size_t rank); + const arma::vec& ratings, double mu, size_t rank); //! Special Initialization constructor. SparseMCLossFunction(const arma::sp_mat& dataset, double mu, size_t rank); diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index d81eb9070a..594ea6bf70 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -44,17 +44,16 @@ BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) size_t threadsAvailable = omp_get_max_threads(); size_t batchSize = std::ceil((float) f.NumFunctions() / threadsAvailable); - ParallelSGD s(f, 10000, batchSize, 1e-5, - decayPolicy); + ParallelSGD s(10000, batchSize, 1e-5, decayPolicy); arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(coordinates); + double result = s.Optimize(f, coordinates); - // The final value of the objective funtion should be close to the optimal - // value, that is the sum of values at the vertices of the parabolae. + // The final value of the objective function should be close to the optimal + // value, that is the sum of values at the vertices of the parabolas. BOOST_REQUIRE_CLOSE(result, 123.75, 0.01); - // The co-ordinates should be the vertices of the parabolae. + // The co-ordinates should be the vertices of the parabolas. BOOST_REQUIRE_CLOSE(coordinates[0], 2, 0.02); BOOST_REQUIRE_CLOSE(coordinates[1], 1, 0.02); BOOST_REQUIRE_CLOSE(coordinates[2], 1.5, 0.02); From 740ae8e8e97c065a01d86f0104f87b9504e28504 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 3 Jul 2017 19:36:45 +0530 Subject: [PATCH 21/41] Remove mean from calculations SparseMCLossFunction --- .../parallel_sgd/sparse_mc_function_impl.hpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp index 8e2aa65a0f..19924b8d3d 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp @@ -71,14 +71,14 @@ double SparseMCLossFunction::Evaluate(const arma::mat& weights, size_t id) // The decision variable is expected to be stored as follows. // The first numRows columns have the first factor matrix, the next numCols // columns have the second factor matrix. The decision variable matrix is - // thus of size (numRows + numCols) x rank. + // thus of size rank x (numCols + numRows). size_t colId = numRows + cols(id); size_t rowId = rows(id); - float error = arma::dot(weights.col(rowId), weights.col(colId)) + - meanRating - ratings(id); - float loss = error * error; + double error = arma::dot(weights.col(rowId), weights.col(colId)) - + ratings(id); + double loss = error * error; // Add the regularisation term. if (rowCnt(rows(id)) > 1) @@ -97,10 +97,9 @@ void SparseMCLossFunction::Gradient(const arma::mat& weights, size_t id, size_t colId = numRows + cols(id); size_t rowId = rows(id); - gradient = arma::sp_mat(numRows + numCols, rank); - - double error = arma::dot(weights.col(rowId), weights.col(colId)) - + meanRating - ratings(id); + gradient = arma::sp_mat(rank, numCols + numRows); + double error = arma::dot(weights.col(rowId), weights.col(colId)) - + ratings(id); // Calculate gradient for the first factor. // Add the regularisation term. From e0e76aa816fa58c4d098f0e07b452b1d58865857 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Tue, 4 Jul 2017 02:28:13 +0530 Subject: [PATCH 22/41] Improve comments in sparse MC implementation. Add another method for testing Parallel SGD. --- .../core/optimizers/lbfgs/test_functions.cpp | 11 +++++ .../core/optimizers/lbfgs/test_functions.hpp | 4 ++ .../optimizers/parallel_sgd/parallel_sgd.hpp | 4 +- .../parallel_sgd/parallel_sgd_impl.hpp | 2 +- .../parallel_sgd/sparse_mc_function.hpp | 44 +++++++++++++++---- .../parallel_sgd/sparse_mc_function_impl.hpp | 14 ++++-- src/mlpack/tests/parallel_sgd_test.cpp | 28 ++++++++++++ 7 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/mlpack/core/optimizers/lbfgs/test_functions.cpp b/src/mlpack/core/optimizers/lbfgs/test_functions.cpp index d60b99c3f6..c109940390 100644 --- a/src/mlpack/core/optimizers/lbfgs/test_functions.cpp +++ b/src/mlpack/core/optimizers/lbfgs/test_functions.cpp @@ -196,6 +196,17 @@ void GeneralizedRosenbrockFunction::Gradient(const arma::mat& coordinates, gradient[i + 1] = 200 * (coordinates[i + 1] - std::pow(coordinates[i], 2)); } +void GeneralizedRosenbrockFunction::Gradient(const arma::mat& coordinates, + const size_t i, + arma::sp_mat& gradient) const +{ + gradient.set_size(n); + + gradient[i] = 400 * (std::pow(coordinates[i], 3) - coordinates[i] * + coordinates[i + 1]) + 2 * (coordinates[i] - 1); + gradient[i + 1] = 200 * (coordinates[i + 1] - std::pow(coordinates[i], 2)); +} + const arma::mat& GeneralizedRosenbrockFunction::GetInitialPoint() const { return initialPoint; diff --git a/src/mlpack/core/optimizers/lbfgs/test_functions.hpp b/src/mlpack/core/optimizers/lbfgs/test_functions.hpp index b44844e072..27251f4de7 100644 --- a/src/mlpack/core/optimizers/lbfgs/test_functions.hpp +++ b/src/mlpack/core/optimizers/lbfgs/test_functions.hpp @@ -129,6 +129,10 @@ class GeneralizedRosenbrockFunction const size_t i, arma::mat& gradient) const; + void Gradient(const arma::mat& coordinates, + const size_t i, + arma::sp_mat& gradient) const; + const arma::mat& GetInitialPoint() const; private: diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index fd8cf4b736..9072d79064 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -25,8 +25,8 @@ namespace optimization { * @misc{1106.5730, * Author = {Feng Niu and Benjamin Recht and Christopher Re and Stephen J. * Wright}, - * Title = {HOGWILD!: A Lock-Free Approach to Parallelizing Stochastic Gradient - * Descent}, + * Title = {HOGWILD!: A Lock-Free Approach to Parallelizing Stochastic + * Gradient Descent}, * Year = {2011}, * Eprint = {arXiv:1106.5730}, * } diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 34ce9f773b..ff88011823 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -39,7 +39,7 @@ double ParallelSGD::Optimize( double overallObjective = 0; double lastObjective = DBL_MAX; - for (size_t i = 1; i <= maxIterations; ++i){ + for (size_t i = 1; i != maxIterations; ++i){ overallObjective = 0; // Get the stepsize for this iteration diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp index ec40baabb4..597c8beb11 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp @@ -24,18 +24,49 @@ class SparseMCLossFunction{ //! Nothing to do for the default constructor. SparseMCLossFunction() = default; - //! Member initialization constructor. + /** + * Member initialization constructor. + * + * @param rows The row indices of the data points. + * @param cols The column indices of the data points. + * @param ratings The ratings of the data points. + * @param mu The regularization parameter. + * @param rank The width of the first factor. + */ SparseMCLossFunction(const arma::uvec& rows, const arma::uvec& cols, const arma::vec& ratings, double mu, size_t rank); - //! Special Initialization constructor. + /** + * Special initialization constructor. + * + * @param dataset The sparse matrix containing the datapoints. + * @param mu The regularization parameter. + * @param rank The width of the first factor. + */ SparseMCLossFunction(const arma::sp_mat& dataset, double mu, size_t rank); - //! Evaluate a function. + /** + * Evaluate the squared error function with the given parameters at the id-th + * data point. + * + * @param weights The decision variable at which the function is to be + * evaluated. + * @param id Index of point to use for objective function evaluation. + * @return The value of the loss function at the given parameter. + */ double Evaluate(const arma::mat& weights, size_t id); - //! Evaluate the gradient of a function. - void Gradient(const arma::mat& weights, size_t id, arma::sp_mat& gradient); + /** + * Evaluate the gradient of the squared error with the given parameters. + * + * @tparam GradType The type of the gradient parameter. + * @param weights The decision variable at which the gradient is to be + * evaluated. + * @param id Index of point to use for objective function evaluation. + * @param gradient Out param for the gradient. + */ + template + void Gradient(const arma::mat& weights, size_t id, GradType& gradient); //! Get the height of the sparse matrix. size_t NumRows() const { return numRows; } @@ -80,9 +111,6 @@ class SparseMCLossFunction{ //! The frequency of the rows. arma::uvec rowCnt; - //! Mean rating. - double meanRating; - //! The regularization parameter. double mu; diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp index 19924b8d3d..1145af4489 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp @@ -63,7 +63,6 @@ void SparseMCLossFunction::CalculateStatistics() rowCnt(rows(i))++; colCnt(cols(i))++; } - meanRating = arma::mean(ratings); } double SparseMCLossFunction::Evaluate(const arma::mat& weights, size_t id) @@ -82,15 +81,22 @@ double SparseMCLossFunction::Evaluate(const arma::mat& weights, size_t id) // Add the regularisation term. if (rowCnt(rows(id)) > 1) - loss += mu * arma::norm(weights.col(rowId)) / (2 * (rowCnt(rows(id)) - 1)); + { + double rowNorm = arma::norm(weights.col(rowId)); + loss += (mu * rowNorm * rowNorm) / (2 * (rowCnt(rows(id)) - 1)); + } if (colCnt(cols(id)) > 1) - loss += mu * arma::norm(weights.col(colId)) / (2 * (colCnt(cols(id)) - 1)); + { + double colNorm = arma::norm(weights.col(colId)); + loss += (mu * colNorm * colNorm) / (2 * (colCnt(cols(id)) - 1)); + } return loss; } +template void SparseMCLossFunction::Gradient(const arma::mat& weights, size_t id, - arma::sp_mat& gradient) + GradType& gradient) { // Index of the column corresponding to the row and column of the current // example in the decision variable. diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index 594ea6bf70..9ce95422ee 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "test_tools.hpp" @@ -60,6 +61,33 @@ BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) BOOST_REQUIRE_CLOSE(coordinates[3], 4, 0.02); } +/** + * When run with a single thread, parallel SGD should be identical to normal + * SGD. + */ +BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) +{ + // Loop over several variants. + for (size_t i = 10; i < 50; i += 5) + { + // Create the generalized Rosenbrock function. + GeneralizedRosenbrockFunction f(i); + + ConstantStep decayPolicy(0.001); + + ParallelSGD s(0, f.NumFunctions(), 1e-12, decayPolicy); + + arma::mat coordinates = f.GetInitialPoint(); + + omp_set_num_threads(1); + double result = s.Optimize(f, coordinates); + + BOOST_REQUIRE_SMALL(result, 1e-8); + for (size_t j = 0; j < i; ++j) + BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 0.01); + } +} + /** * Test the correctness of the Exponential backoff stepsize decay policy. */ From c9323ee4b091992bebfbf064b8930a7ee5aae914 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Tue, 4 Jul 2017 03:20:53 +0530 Subject: [PATCH 23/41] Test for ParallelSGD::ThreadShare Update comment and follow variable naming convention --- .../optimizers/parallel_sgd/parallel_sgd.hpp | 2 +- .../parallel_sgd/parallel_sgd_impl.hpp | 12 +++--- .../parallel_sgd/sparse_test_function.hpp | 2 +- src/mlpack/tests/parallel_sgd_test.cpp | 41 ++++++++++++++++++- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 9072d79064..8265535b61 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -132,7 +132,7 @@ class ParallelSGD * iteration. * @return Vector of datapoint indices to be visited by the current thread. */ - arma::Col ThreadShare(size_t thread_id, + arma::Col ThreadShare(size_t threadId, const arma::Col& visitationOrder); //! The maximum number of allowed iterations. diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index ff88011823..ab4b5a8a4a 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -109,24 +109,24 @@ void ParallelSGD::GenerateVisitationOrder( template arma::Col ParallelSGD::ThreadShare( - size_t thread_id, const arma::Col& visitationOrder) + size_t threadId, const arma::Col& visitationOrder) { - if (thread_id * batchSize >= visitationOrder.n_elem) + if (threadId * batchSize >= visitationOrder.n_elem) { // No data for this thread. return arma::Col(); } - else if ((thread_id + 1) * batchSize >= visitationOrder.n_elem) + else if ((threadId + 1) * batchSize >= visitationOrder.n_elem) { // The last few elements. - return visitationOrder.subvec(thread_id * batchSize, + return visitationOrder.subvec(threadId * batchSize, visitationOrder.n_elem - 1); } else { // Equal distribution of batchSize examples to each thread. - return visitationOrder.subvec(thread_id * batchSize, - (thread_id + 1) * batchSize - 1); + return visitationOrder.subvec(threadId * batchSize, + (threadId + 1) * batchSize - 1); } } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp index 2f48cd466d..fb48a49efc 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp @@ -29,7 +29,7 @@ class SparseTestFunction //! Set members in the default constructor. SparseTestFunction(); - //! Return 6 (the number of functions). + //! Return 4 (the number of functions). size_t NumFunctions() const { return 4; } //! Get the starting point. diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index 9ce95422ee..ac7dacd012 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -10,11 +10,14 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include -#include #include #include #include #include +// We need some thorough testing +#define private public +#include +#undef private #include #include "test_tools.hpp" @@ -88,6 +91,42 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) } } +/** + * Test if the data points are divided correctly among the threads. + */ +BOOST_AUTO_TEST_CASE(ThreadSharingTest) +{ + ConstantStep decayPolicy(0); + + // Each thread gets a batch of size 4. + ParallelSGD s(0, 4, 1e-10, decayPolicy); + + // Generate a random visitation order. + arma::Col visitationOrder; + s.GenerateVisitationOrder(visitationOrder, 10); + + // Lets count how many times each example is handed out in an iteration. + arma::Col count(10, arma::fill::zeros); + + for (size_t threadId = 0; threadId < 4; ++threadId) + { + arma::Col share = s.ThreadShare(threadId, visitationOrder); + for (size_t i = 0; i < share.n_elem; ++i) + count(share(i))++; + + // The last thread to have some data. + if (threadId == 2) + BOOST_REQUIRE_EQUAL(share.n_elem, 2); + + // Only the first 3 threads get data. + if (threadId > 2) + BOOST_REQUIRE_EQUAL(share.n_elem, 0); + } + + // If everything is correct, each count should be 1 for each data point. + CheckMatrices(count, arma::Col(10, arma::fill::ones)); +} + /** * Test the correctness of the Exponential backoff stepsize decay policy. */ From f2883c22a3fb1288247dc5507f10e06115da9e99 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Wed, 5 Jul 2017 01:00:34 +0530 Subject: [PATCH 24/41] Vary the number of threads while testing in SimpleParallelSGDTest --- src/mlpack/tests/parallel_sgd_test.cpp | 30 +++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index ac7dacd012..06043c8ebc 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -46,22 +46,28 @@ BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) // test will fail. size_t threadsAvailable = omp_get_max_threads(); - size_t batchSize = std::ceil((float) f.NumFunctions() / threadsAvailable); - ParallelSGD s(10000, batchSize, 1e-5, decayPolicy); + for (size_t i = threadsAvailable; i > 0; --i) + { + omp_set_num_threads(i); - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); + size_t batchSize = std::ceil((float) f.NumFunctions() / i); - // The final value of the objective function should be close to the optimal - // value, that is the sum of values at the vertices of the parabolas. - BOOST_REQUIRE_CLOSE(result, 123.75, 0.01); + ParallelSGD s(10000, batchSize, 1e-5, decayPolicy); - // The co-ordinates should be the vertices of the parabolas. - BOOST_REQUIRE_CLOSE(coordinates[0], 2, 0.02); - BOOST_REQUIRE_CLOSE(coordinates[1], 1, 0.02); - BOOST_REQUIRE_CLOSE(coordinates[2], 1.5, 0.02); - BOOST_REQUIRE_CLOSE(coordinates[3], 4, 0.02); + arma::mat coordinates = f.GetInitialPoint(); + double result = s.Optimize(f, coordinates); + + // The final value of the objective function should be close to the optimal + // value, that is the sum of values at the vertices of the parabolas. + BOOST_REQUIRE_CLOSE(result, 123.75, 0.01); + + // The co-ordinates should be the vertices of the parabolas. + BOOST_REQUIRE_CLOSE(coordinates[0], 2, 0.02); + BOOST_REQUIRE_CLOSE(coordinates[1], 1, 0.02); + BOOST_REQUIRE_CLOSE(coordinates[2], 1.5, 0.02); + BOOST_REQUIRE_CLOSE(coordinates[3], 4, 0.02); + } } /** From b69e1dfe43b2b3232b6291f5538c89e1a0af68ae Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sun, 9 Jul 2017 01:58:01 +0530 Subject: [PATCH 25/41] Add Recover function to sparse matrix completion --- .../core/optimizers/parallel_sgd/parallel_sgd_impl.hpp | 2 +- .../core/optimizers/parallel_sgd/sparse_mc_function.hpp | 8 ++++++++ .../optimizers/parallel_sgd/sparse_mc_function_impl.hpp | 6 ++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index ab4b5a8a4a..0efc90584e 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -72,7 +72,7 @@ double ParallelSGD::Optimize( ++cur) { #pragma omp atomic - iterate(cur.row(), i) -= stepSize * gradient(cur.row(), i); + iterate(cur.row(), i) -= stepSize * (*cur); } } } diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp index 597c8beb11..e13aa19c9c 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp @@ -68,6 +68,14 @@ class SparseMCLossFunction{ template void Gradient(const arma::mat& weights, size_t id, GradType& gradient); + /** + * Get the recovered matrix from the iterate. + * @param weights The decision variable at which the gradient is to be + * evaluated. + * @return The recovered matrix after completion. + */ + arma::mat Recover(const arma::mat& weights); + //! Get the height of the sparse matrix. size_t NumRows() const { return numRows; } //! Modify the height of the sparse matrix. diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp index 1145af4489..6c52120ab1 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp @@ -122,6 +122,12 @@ void SparseMCLossFunction::Gradient(const arma::mat& weights, size_t id, gradient.col(colId) += error * weights.col(rowId); } +arma::mat SparseMCLossFunction::Recover(const arma::mat& weights) +{ + return arma::trans(weights.cols(0, numRows - 1)) * weights.cols(numRows, + numRows + numCols - 1); +} + size_t SparseMCLossFunction::NumFunctions() { return rows.n_elem; From 8fab2121e41623f967afeba7783024e8a44fdd29 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Thu, 13 Jul 2017 15:42:31 +0530 Subject: [PATCH 26/41] Refactor regularized SVD for sparse applications --- .../optimizers/parallel_sgd/CMakeLists.txt | 2 - .../parallel_sgd/sparse_svm_function.hpp | 55 ------------- .../parallel_sgd/sparse_svm_function_impl.hpp | 42 ---------- .../regularized_svd_function.cpp | 45 +++++++++-- .../regularized_svd_function.hpp | 24 +++++- src/mlpack/methods/sparse_svm/CMakeLists.txt | 16 ++++ .../sparse_svm/sparse_svm_function.hpp | 77 +++++++++++++++++++ .../sparse_svm/sparse_svm_function_impl.hpp | 46 +++++++++++ 8 files changed, 199 insertions(+), 108 deletions(-) delete mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp delete mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp create mode 100644 src/mlpack/methods/sparse_svm/CMakeLists.txt create mode 100644 src/mlpack/methods/sparse_svm/sparse_svm_function.hpp create mode 100644 src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp diff --git a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt index ef9ea07c48..9a62144ca5 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt +++ b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt @@ -1,8 +1,6 @@ set(SOURCES parallel_sgd.hpp parallel_sgd_impl.hpp - sparse_svm_function.hpp - sparse_svm_function_impl.hpp sparse_mc_function.hpp sparse_mc_function_impl.hpp ) diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp deleted file mode 100644 index 76f0582314..0000000000 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/** - * @file sparse_svm_function.hpp - * @author Shikhar Bhardwaj - * - * Implementation of the hinge loss function for training a sparse SVM with the - * parallel SGD algorithm - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_SVM_HPP -#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_SVM_HPP -#include - -class SparseSVMLossFunction{ - public: - //! Nothing to do for the default constructor. - SparseSVMLossFunction() = default; - - //! Member initialization constructor. - SparseSVMLossFunction(arma::sp_mat& dataset, arma::vec& labels); - - //! Evaluate a function. - double Evaluate(const arma::mat& weights, size_t id); - - //! Evaluate the gradient of a function. - void Gradient(const arma::mat& weights, size_t id, arma::sp_mat& gradient); - - //! Get the dataset. - const arma::sp_mat& Dataset() const { return dataset; } - //! Modify the dataset. - arma::sp_mat& Dataset() { return dataset; } - - //! Get the labels. - const arma::vec& Labels() const { return labels; } - //! Modify the labels. - arma::vec& Labels() { return labels; } - - //! Return the number of functions. - size_t NumFunctions(); - - private: - //! The datapoints for training. - arma::sp_mat dataset; - - //! The labels, y_i. - arma::vec labels; -}; - -// Include implementation -#include "sparse_svm_function_impl.hpp" - -#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp deleted file mode 100644 index 386d2937c9..0000000000 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_svm_function_impl.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/** - * @file sparse_svm_function_impl.hpp - * @author Shikhar Bhardwaj - * - * Implementation of the hinge loss function for training a sparse SVM with the - * parallel SGD algorithm - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_SVM_IMPL_HPP -#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_SVM_IMPL_HPP - -// In case it hasn't been included yet. -#include "sparse_svm_function.hpp" - -SparseSVMLossFunction::SparseSVMLossFunction( - arma::sp_mat& dataset, arma::vec& labels) : - dataset(dataset), labels(labels) -{ /* Nothing to do */ } - -double SparseSVMLossFunction::Evaluate(const arma::mat& weights, size_t id) -{ - return std::max(0.0, 1 - labels(id) * arma::dot(dataset.col(id), weights)); -} - -void SparseSVMLossFunction::Gradient( - const arma::mat& weights, size_t id, arma::sp_mat& gradient) -{ - double dot = 1 - labels(id) * arma::dot(weights, dataset.col(id)); - gradient = (dot < 0) ? arma::sp_mat(weights.n_rows, 1) : - (-1 * arma::sp_mat(dataset.col(id) * labels(id))); -} - -size_t SparseSVMLossFunction::NumFunctions() -{ - return dataset.n_cols; -} - -#endif diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp index 2f22ea8cba..1de4f3c81d 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp @@ -16,9 +16,10 @@ namespace mlpack { namespace svd { -RegularizedSVDFunction::RegularizedSVDFunction(const arma::mat& data, - const size_t rank, - const double lambda) : +template +RegularizedSVDFunction::RegularizedSVDFunction(const MatType& data, + const size_t rank, + const double lambda) : data(data), rank(rank), lambda(lambda) @@ -31,7 +32,9 @@ RegularizedSVDFunction::RegularizedSVDFunction(const arma::mat& data, initialPoint.randu(rank, numUsers + numItems); } -double RegularizedSVDFunction::Evaluate(const arma::mat& parameters) const +template +double RegularizedSVDFunction::Evaluate(const arma::mat& parameters) +const { // The cost for the optimization is as follows: // f(u, v) = sum((rating(i, j) - u(i).t() * v(j))^2) @@ -66,8 +69,9 @@ double RegularizedSVDFunction::Evaluate(const arma::mat& parameters) const return cost; } -double RegularizedSVDFunction::Evaluate(const arma::mat& parameters, - const size_t i) const +template +double RegularizedSVDFunction::Evaluate(const arma::mat& parameters, + const size_t i) const { // Indices for accessing the the correct parameter columns. const size_t user = data(0, i); @@ -88,8 +92,9 @@ double RegularizedSVDFunction::Evaluate(const arma::mat& parameters, return (ratingErrorSquared + regularizationError); } -void RegularizedSVDFunction::Gradient(const arma::mat& parameters, - arma::mat& gradient) const +template +void RegularizedSVDFunction::Gradient(const arma::mat& parameters, + arma::mat& gradient) const { // For an example with rating corresponding to user 'i' and item 'j', the // gradients for the parameters is as follows: @@ -122,6 +127,30 @@ void RegularizedSVDFunction::Gradient(const arma::mat& parameters, } } +template +template +void RegularizedSVDFunction::Gradient(const arma::mat ¶meters, + size_t id, + GradType &gradient) const +{ + gradient.zeros(rank, numUsers + numItems); + + const size_t user = data(0, id); + const size_t item = data(1, id) + numUsers; + + // Prediction error for the example. + const double rating = data(2, id); + double ratingError = rating - arma::dot(parameters.col(user), + parameters.col(item)); + + // Gradient is non-zero only for the parameter columns corresponding to the + // example. + gradient.col(user) += 2 * (lambda * parameters.col(user) - + ratingError * parameters.col(item)); + gradient.col(item) += 2 * (lambda * parameters.col(item) - + ratingError * parameters.col(user)); +} + } // namespace svd } // namespace mlpack diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp index ef20bf3134..7791a8ad89 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp @@ -19,6 +19,13 @@ namespace mlpack { namespace svd { +/** + * The data is stored in a matrix of type MatType, so that this class can be + * used with both dense and sparse matrix types. + * + * @tparam MatType The matrix type of the dataset. + */ +template class RegularizedSVDFunction { public: @@ -31,7 +38,7 @@ class RegularizedSVDFunction * @param rank Rank used for matrix factorization. * @param lambda Regularization parameter used for optimization. */ - RegularizedSVDFunction(const arma::mat& data, + RegularizedSVDFunction(const MatType& data, const size_t rank, const double lambda); @@ -62,6 +69,21 @@ class RegularizedSVDFunction void Gradient(const arma::mat& parameters, arma::mat& gradient) const; + /** + * Evaluates the gradient of the cost function over one training example. + * This function is useful for optimizers like SGD. The type of the gradient + * parameter is a template to allow the computation of a sparse gradient. + * + * @tparam GradType The type of the gradient out-param. + * @param parameters Parameters(user/item matrices) of the decomposition. + * @param id The index of the training example. + * @param gradient Calculated gradient for the parameters. + */ + template + void Gradient(const arma::mat& parameters, + size_t id, + GradType& gradient) const; + //! Return the initial point for the optimization. const arma::mat& GetInitialPoint() const { return initialPoint; } diff --git a/src/mlpack/methods/sparse_svm/CMakeLists.txt b/src/mlpack/methods/sparse_svm/CMakeLists.txt new file mode 100644 index 0000000000..04c06b571b --- /dev/null +++ b/src/mlpack/methods/sparse_svm/CMakeLists.txt @@ -0,0 +1,16 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into the output library +# Do not include test programs here +set(SOURCES + sparse_svm_function.hpp + sparse_svm_function_impl.hpp +) + +# add directory name to sources +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# append sources (with directory name) to list of all mlpack sources (used at +# the parent scope) +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp b/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp new file mode 100644 index 0000000000..fe17c0838f --- /dev/null +++ b/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp @@ -0,0 +1,77 @@ +/** + * @file sparse_svm_function.hpp + * @author Shikhar Bhardwaj + * + * Implementation of the hinge loss function for training a sparse SVM with the + * parallel SGD algorithm + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_SPARSE_SVM_SPARSE_SVM_FUNCTION_HPP +#define MLPACK_METHODS_SPARSE_SVM_SPARSE_SVM_FUNCTION_HPP + +#include + +class SparseSVMFunction{ + public: + //! Nothing to do for the default constructor. + SparseSVMFunction() {} + + //! Member initialization constructor. + SparseSVMFunction(const arma::sp_mat& dataset, const arma::vec& labels); + + /** + * Evaluate the hinge loss function on the specified datapoint. + * + * @param parameters The parameters of the SVM. + * @param id Index of the datapoint to use for function evaluation. + * @return The value of the loss function at the given parameters. + */ + double Evaluate(const arma::mat& parameters, size_t id); + + /** + * Evaluate the gradient the gradient of the hinge loss function, following + * the SparseFunctionType requirements on the Gradient function. + * + * @param parameters The parameters of the SVM. + * @param id Index of the datapoint to use for the gradient evaluation. + * @param gradient Sparse matrix to output the gradient into. + */ + void Gradient(const arma::mat& parameters, size_t id, arma::sp_mat& gradient); + + //! Return the initial point for the optimization. + const arma::mat& InitialPoint() const { return initialPoint; } + //! Modify the initial point for the optimization. + arma::mat& InitialPoint() { return initialPoint; } + + //! Get the dataset. + const arma::sp_mat& Dataset() const { return dataset; } + //! Modify the dataset. + arma::sp_mat& Dataset() { return dataset; } + + //! Get the labels. + const arma::vec& Labels() const { return labels; } + //! Modify the labels. + arma::vec& Labels() { return labels; } + + //! Return the number of functions. + size_t NumFunctions(); + + private: + //! The initial point, from which to start the optimization. + arma::mat initialPoint; + + //! The datapoints for training. + arma::sp_mat dataset; + + //! The labels, y_i. + arma::vec labels; +}; + +// Include implementation +#include "sparse_svm_function_impl.hpp" + +#endif // MLPACK_METHODS_SPARSE_SVM_SPARSE_SVM_FUNCTION_HPP diff --git a/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp b/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp new file mode 100644 index 0000000000..5150abdda4 --- /dev/null +++ b/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp @@ -0,0 +1,46 @@ +/** + * @file sparse_svm_function_impl.hpp + * @author Shikhar Bhardwaj + * + * Implementation of the hinge loss function for training a sparse SVM with the + * parallel SGD algorithm + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_SPARSE_SVM_SPARSE_SVM_FUNCTION_IMPL_HPP +#define MLPACK_METHODS_SPARSE_SVM_SPARSE_SVM_FUNCTION_IMPL_HPP + +// In case it hasn't been included yet. +#include "sparse_svm_function.hpp" + +SparseSVMFunction::SparseSVMFunction( + const arma::sp_mat& dataset, const arma::vec& labels) : + dataset(dataset), labels(labels) +{ /* Nothing to do */ } + +double SparseSVMFunction::Evaluate(const arma::mat& parameters, size_t id) +{ + // The hinge loss function. + return std::max(0.0, 1 - labels(id) * arma::dot(dataset.col(id), parameters)); +} + +void SparseSVMFunction::Gradient( + const arma::mat& parameters, size_t id, arma::sp_mat& gradient) +{ + // Evaluate the gradient of the hinge loss function. + double dot = 1 - labels(id) * arma::dot(parameters, dataset.col(id)); + gradient = (dot < 0) ? arma::sp_mat(parameters.n_rows, 1) : + (-1 * arma::sp_mat(dataset.col(id) * labels(id))); +} + +size_t SparseSVMFunction::NumFunctions() +{ + // The number of points in the dataset is the number of functions, as this + // is a data dependent function. + return dataset.n_cols; +} + +#endif // MLPACK_METHODS_SPARSE_SVM_SPARSE_SVM_FUNCTION_IMPL_HPP From 9e5fc1c04cc07164557624a098688ab18c8fb970 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Thu, 13 Jul 2017 19:16:05 +0530 Subject: [PATCH 27/41] Refactor regularized SVD to have required overloads for ParallelSGD --- .../methods/regularized_svd/CMakeLists.txt | 2 +- .../regularized_svd_function.hpp | 12 +-- ....cpp => regularized_svd_function_impl.hpp} | 13 +++- .../regularized_svd/regularized_svd_impl.hpp | 2 +- src/mlpack/tests/regularized_svd_test.cpp | 75 +++++++++++++++++-- 5 files changed, 86 insertions(+), 18 deletions(-) rename src/mlpack/methods/regularized_svd/{regularized_svd_function.cpp => regularized_svd_function_impl.hpp} (97%) diff --git a/src/mlpack/methods/regularized_svd/CMakeLists.txt b/src/mlpack/methods/regularized_svd/CMakeLists.txt index 5953f055db..106fb194bb 100644 --- a/src/mlpack/methods/regularized_svd/CMakeLists.txt +++ b/src/mlpack/methods/regularized_svd/CMakeLists.txt @@ -4,7 +4,7 @@ set(SOURCES regularized_svd.hpp regularized_svd_impl.hpp regularized_svd_function.hpp - regularized_svd_function.cpp + regularized_svd_function_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp index 706fc4be11..da44105aa6 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp @@ -107,7 +107,7 @@ class RegularizedSVDFunction private: //! Rating data. - const arma::mat& data; + const MatType& data; //! Initial parameter point. arma::mat initialPoint; //! Rank used for matrix factorization. @@ -131,13 +131,15 @@ namespace optimization { * affects only a small number of parameters per example, and thus the normal * abstraction does not work as fast as we might like it to. */ - template<> - template<> - double StandardSGD::Optimize( - mlpack::svd::RegularizedSVDFunction& function, + template <> + template <> + inline double StandardSGD::Optimize( + mlpack::svd::RegularizedSVDFunction& function, arma::mat& parameters); } // namespace optimization } // namespace mlpack +#include "regularized_svd_function_impl.hpp" + #endif diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp similarity index 97% rename from src/mlpack/methods/regularized_svd/regularized_svd_function.cpp rename to src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp index 8397caf0c0..31200b681c 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp @@ -10,8 +10,11 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ + +#ifndef MLPACK_METHODS_REGULARIZED_SVD_REGULARIZED_FUNCTION_SVD_IMPL_HPP +#define MLPACK_METHODS_REGULARIZED_SVD_REGULARIZED_FUNCTION_SVD_IMPL_HPP + #include "regularized_svd_function.hpp" -#include namespace mlpack { namespace svd { @@ -158,10 +161,10 @@ void RegularizedSVDFunction::Gradient(const arma::mat ¶meters, namespace mlpack { namespace optimization { -template<> -template<> +template <> +template <> double StandardSGD::Optimize( - mlpack::svd::RegularizedSVDFunction& function, + mlpack::svd::RegularizedSVDFunction& function, arma::mat& parameters) { // Find the number of functions to use. @@ -217,3 +220,5 @@ double StandardSGD::Optimize( } // namespace optimization } // namespace mlpack + +#endif diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_impl.hpp index b8e0c10734..b411a1920c 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_impl.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_impl.hpp @@ -34,7 +34,7 @@ void RegularizedSVD::Apply(const arma::mat& data, arma::mat& v) { // Make the optimizer object using a RegularizedSVDFunction object. - RegularizedSVDFunction rSVDFunc(data, rank, lambda); + RegularizedSVDFunction rSVDFunc(data, rank, lambda); mlpack::optimization::StandardSGD optimizer(alpha, iterations * data.n_cols); // Get optimized parameters. diff --git a/src/mlpack/tests/regularized_svd_test.cpp b/src/mlpack/tests/regularized_svd_test.cpp index 91c1262203..5c3fce6489 100644 --- a/src/mlpack/tests/regularized_svd_test.cpp +++ b/src/mlpack/tests/regularized_svd_test.cpp @@ -11,12 +11,15 @@ */ #include #include +#include +#include #include #include "test_tools.hpp" using namespace mlpack; using namespace mlpack::svd; +using namespace mlpack::optimization; BOOST_AUTO_TEST_SUITE(RegularizedSVDTest); @@ -41,7 +44,7 @@ BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionRandomEvaluate) data(1, numRatings - 1) = numItems - 1; // Make a RegularizedSVDFunction with zero regularization. - RegularizedSVDFunction rSVDFunc(data, rank, 0); + RegularizedSVDFunction rSVDFunc(data, rank, 0); for (size_t i = 0; i < numTrials; i++) { @@ -89,9 +92,9 @@ BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionRegularizationEvaluate) // Make three RegularizedSVDFunction objects with different amounts of // regularization. - RegularizedSVDFunction rSVDFuncNoReg(data, rank, 0); - RegularizedSVDFunction rSVDFuncSmallReg(data, rank, 0.5); - RegularizedSVDFunction rSVDFuncBigReg(data, rank, 20); + RegularizedSVDFunction rSVDFuncNoReg(data, rank, 0); + RegularizedSVDFunction rSVDFuncSmallReg(data, rank, 0.5); + RegularizedSVDFunction rSVDFuncBigReg(data, rank, 20); for (size_t i = 0; i < numTrials; i++) { @@ -146,8 +149,8 @@ BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionGradient) // Make two RegularizedSVDFunction objects, one with regularization and one // without. - RegularizedSVDFunction rSVDFunc1(data, rank, 0); - RegularizedSVDFunction rSVDFunc2(data, rank, 0.5); + RegularizedSVDFunction rSVDFunc1(data, rank, 0); + RegularizedSVDFunction rSVDFunc2(data, rank, 0.5); // Calculate gradients for both the objects. arma::mat gradient1, gradient2; @@ -225,7 +228,7 @@ BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionOptimize) } // Make the Reg SVD function and the optimizer. - RegularizedSVDFunction rSVDFunc(data, rank, lambda); + RegularizedSVDFunction rSVDFunc(data, rank, lambda); mlpack::optimization::StandardSGD optimizer(alpha, iterations * numRatings); // Obtain optimized parameters after training. @@ -248,4 +251,62 @@ BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionOptimize) BOOST_REQUIRE_SMALL(relativeError, 1e-2); } +// Test Regularized SVD with parallel SGD. +BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionOptimizeHOGWILD) +{ + // Define useful constants. + const size_t numUsers = 50; + const size_t numItems = 50; + const size_t numRatings = 100; + const size_t rank = 10; + const double alpha = 0.01; + const double lambda = 0.01; + + // Initiate random parameters. + arma::mat parameters = arma::randu(rank, numUsers + numItems); + + // Make a random rating dataset. + arma::mat data = arma::randu(3, numRatings); + data.row(0) = floor(data.row(0) * numUsers); + data.row(1) = floor(data.row(1) * numItems); + + // Manually set last row to maximum user and maximum item. + data(0, numRatings - 1) = numUsers - 1; + data(1, numRatings - 1) = numItems - 1; + + // Make rating entries based on the parameters. + for (size_t i = 0; i < numRatings; i++) + { + data(2, i) = arma::dot(parameters.col(data(0, i)), + parameters.col(numUsers + data(1, i))); + } + + // Make the Reg SVD function and the optimizer. + RegularizedSVDFunction rSVDFunc(data, rank, lambda); + + ConstantStep decayPolicy(alpha); + + ParallelSGD optimizer(0, + rSVDFunc.NumFunctions() / omp_get_max_threads(), 1e-5, decayPolicy); + + // Obtain optimized parameters after training. + arma::mat optParameters = arma::randu(rank, numUsers + numItems); + optimizer.Optimize(rSVDFunc, optParameters); + + // Get predicted ratings from optimized parameters. + arma::mat predictedData(1, numRatings); + for (size_t i = 0; i < numRatings; i++) + { + predictedData(0, i) = arma::dot(optParameters.col(data(0, i)), + optParameters.col(numUsers + data(1, i))); + } + + // Calculate relative error. + const double relativeError = arma::norm(data.row(2) - predictedData, "frob") / + arma::norm(data, "frob"); + + // Relative error should be small. + BOOST_REQUIRE_SMALL(relativeError, 1e-2); +} + BOOST_AUTO_TEST_SUITE_END(); From 54402c3048eb3065766636eab46b5d4b0310278a Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Thu, 13 Jul 2017 20:34:02 +0530 Subject: [PATCH 28/41] Fix minor style issues and use more meaningful names. --- .../parallel_sgd/decay_policies/constant_step.hpp | 7 ++++--- .../decay_policies/exponential_backoff.hpp | 7 ++++--- .../core/optimizers/parallel_sgd/parallel_sgd.hpp | 12 ++++++------ .../optimizers/parallel_sgd/parallel_sgd_impl.hpp | 14 +++++++------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp index f43876bfde..1c456a9cec 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp @@ -14,13 +14,14 @@ #include -namespace mlpack{ -namespace optimization{ +namespace mlpack { +namespace optimization { /** * Implementation of the ConstantStep stepsize decay policy for parallel SGD. */ -class ConstantStep{ +class ConstantStep +{ public: ConstantStep(double initalStep) : step(initalStep) { /* Nothing to do */ } diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index 184241d47a..4f8b1da01c 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -15,8 +15,8 @@ #include -namespace mlpack{ -namespace optimization{ +namespace mlpack { +namespace optimization { /** * Exponential backoff stepsize reduction policy for parallel SGD. @@ -35,7 +35,8 @@ namespace optimization{ * This stepsize update scheme gives robust 1/k convergence rates to the * implementation of parallel SGD. */ -class ExponentialBackoff{ +class ExponentialBackoff +{ public: /** * Construct the exponential backoff policy with the required parameters. diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 8265535b61..c82cb34ade 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -64,13 +64,13 @@ class ParallelSGD * by each thread. * * @param maxIterations Maximum number of iterations allowed. - * @param batchSize Number of datapoints to be processed in one iteration by - * each thread. + * @param threadShareSize Number of datapoints to be processed in one + * iteration by each thread. * @param tolerance Maximum absolute tolerance to terminate the algorithm. * @param decayPolicy The step size update policy to use. */ ParallelSGD(const size_t maxIterations, - const size_t batchSize, + const size_t threadShareSize, const double tolerance, const DecayPolicyType& decayPolicy); @@ -95,10 +95,10 @@ class ParallelSGD //! Get the number of datapoints to be processed in one iteration by each //! thread. - size_t BatchSize() const { return batchSize; } + size_t ThreadShareSize() const { return threadShareSize; } //! Modify the number of datapoints to be processed in one iteration by each //! thread. - size_t& BatchSize() { return batchSize; } + size_t& ThreadShareSize() { return threadShareSize; } //! Get the tolerance for termination. double Tolerance() const { return tolerance; } @@ -139,7 +139,7 @@ class ParallelSGD size_t maxIterations; //! The number of datapoints to be processed in one iteration by each thread. - size_t batchSize; + size_t threadShareSize; //! The tolerance for termination. double tolerance; diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 0efc90584e..5786ff56e6 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -21,11 +21,11 @@ namespace optimization { template ParallelSGD::ParallelSGD( const size_t maxIterations, - const size_t batchSize, + const size_t threadShareSize, const double tolerance, const DecayPolicyType& decayPolicy) : maxIterations(maxIterations), - batchSize(batchSize), + threadShareSize(threadShareSize), tolerance(tolerance), decayPolicy(decayPolicy) { /* Nothing to do. */ } @@ -111,22 +111,22 @@ template arma::Col ParallelSGD::ThreadShare( size_t threadId, const arma::Col& visitationOrder) { - if (threadId * batchSize >= visitationOrder.n_elem) + if (threadId * threadShareSize >= visitationOrder.n_elem) { // No data for this thread. return arma::Col(); } - else if ((threadId + 1) * batchSize >= visitationOrder.n_elem) + else if ((threadId + 1) * threadShareSize >= visitationOrder.n_elem) { // The last few elements. - return visitationOrder.subvec(threadId * batchSize, + return visitationOrder.subvec(threadId * threadShareSize , visitationOrder.n_elem - 1); } else { // Equal distribution of batchSize examples to each thread. - return visitationOrder.subvec(threadId * batchSize, - (threadId + 1) * batchSize - 1); + return visitationOrder.subvec(threadId * threadShareSize, + (threadId + 1) * threadShareSize - 1); } } From 537500a0b06cd35a00f2d76bd293395fd49f85d3 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Fri, 14 Jul 2017 02:01:03 +0530 Subject: [PATCH 29/41] Refactor functions with DecomposableFunctionType interface to compute sparse gradients --- .../logistic_regression_function.hpp | 7 +++++-- .../logistic_regression_function_impl.hpp | 7 ++++--- src/mlpack/methods/nca/nca_softmax_error_function.hpp | 8 ++++++-- .../methods/nca/nca_softmax_error_function_impl.hpp | 11 ++++++----- .../regularized_svd/regularized_svd_function.hpp | 3 ++- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp index 586d69eed7..2b6f2cb83a 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp @@ -94,15 +94,18 @@ class LogisticRegressionFunction * Evaluate the gradient of the logistic regression log-likelihood function * with the given parameters, and with respect to only one point in the * dataset. This is useful for optimizers such as SGD, which require a - * separable objective function. + * separable objective function. The type of the gradient parameter is a + * template argument to allow the computation of a sparse gradient. * + * @tparam GradType The type of the gradient out-param. * @param parameters Vector of logistic regression parameters. * @param i Index of points to use for objective function gradient evaluation. * @param gradient Vector to output gradient into. */ + template void Gradient(const arma::mat& parameters, const size_t i, - arma::mat& gradient) const; + GradType& gradient) const; //! Return the initial point for the optimization. const arma::mat& GetInitialPoint() const { return initialPoint; } diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp index 53b1a953b8..7a2f6f64ee 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp @@ -149,14 +149,15 @@ void LogisticRegressionFunction::Gradient( * function with respect to individual points. This is useful for optimizers * that use a separable objective function, such as SGD. */ -template +template +template void LogisticRegressionFunction::Gradient( const arma::mat& parameters, const size_t i, - arma::mat& gradient) const + GradType& gradient) const { // Calculate the regularization term. - arma::mat regularization; + GradType regularization; regularization = lambda * parameters.col(0).subvec(1, parameters.n_elem - 1) / predictors.n_cols; diff --git a/src/mlpack/methods/nca/nca_softmax_error_function.hpp b/src/mlpack/methods/nca/nca_softmax_error_function.hpp index 1b3f9bb5e9..151454c354 100644 --- a/src/mlpack/methods/nca/nca_softmax_error_function.hpp +++ b/src/mlpack/methods/nca/nca_softmax_error_function.hpp @@ -14,6 +14,7 @@ #define MLPACK_METHODS_NCA_NCA_SOFTMAX_ERROR_FUNCTION_HPP #include +#include namespace mlpack { namespace nca { @@ -92,15 +93,18 @@ class SoftmaxErrorFunction * matrix on only one point of the dataset. This is the separable * implementation, where the objective function is decomposed into the sum of * many objective functions, and here, only one of those constituent objective - * functions is returned. + * functions is returned. The type of the gradient parameter is a template + * argument to allow the computation of a sparse gradient. * + * @tparam GradType The type of the gradient out-param. * @param covariance Covariance matrix of Mahalanobis distance. * @param i Index of point to use for objective function. * @param gradient Matrix to store the calculated gradient in. */ + template void Gradient(const arma::mat& covariance, const size_t i, - arma::mat& gradient); + GradType& gradient); /** * Get the initial point. diff --git a/src/mlpack/methods/nca/nca_softmax_error_function_impl.hpp b/src/mlpack/methods/nca/nca_softmax_error_function_impl.hpp index dd3bc248e3..b9c1a22bde 100644 --- a/src/mlpack/methods/nca/nca_softmax_error_function_impl.hpp +++ b/src/mlpack/methods/nca/nca_softmax_error_function_impl.hpp @@ -132,10 +132,11 @@ void SoftmaxErrorFunction::Gradient(const arma::mat& coordinates, } //! The separable implementation. -template +template +template void SoftmaxErrorFunction::Gradient(const arma::mat& coordinates, const size_t i, - arma::mat& gradient) + GradType& gradient) { // We will need to calculate p_i before this evaluation is done, so these two // variables will hold the information necessary for that. @@ -144,8 +145,8 @@ void SoftmaxErrorFunction::Gradient(const arma::mat& coordinates, // The gradient involves two matrix terms which are eventually combined into // one. - arma::mat firstTerm; - arma::mat secondTerm; + GradType firstTerm; + GradType secondTerm; firstTerm.zeros(coordinates.n_rows, coordinates.n_cols); secondTerm.zeros(coordinates.n_rows, coordinates.n_cols); @@ -166,7 +167,7 @@ void SoftmaxErrorFunction::Gradient(const arma::mat& coordinates, // If the points are in the same class, we must add to the second term of // the gradient as well as the numerator of p_i. We will divide by the // denominator of p_ik later. For x_ik we are not using stretched points. - arma::vec x_ik = dataset.col(i) - dataset.col(k); + GradType x_ik = dataset.col(i) - dataset.col(k); if (labels[i] == labels[k]) { numerator += eval; diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp index da44105aa6..03e496d347 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp @@ -72,7 +72,8 @@ class RegularizedSVDFunction /** * Evaluates the gradient of the cost function over one training example. * This function is useful for optimizers like SGD. The type of the gradient - * parameter is a template to allow the computation of a sparse gradient. + * parameter is a template argument to allow the computation of a sparse + * gradient. * * @tparam GradType The type of the gradient out-param. * @param parameters Parameters(user/item matrices) of the decomposition. From a54e780e42ded6015e41214aed7e70ec58ed3861 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Fri, 14 Jul 2017 03:16:20 +0530 Subject: [PATCH 30/41] Better logging information, remove sparse matrix completion function --- .../optimizers/parallel_sgd/CMakeLists.txt | 2 - .../parallel_sgd/parallel_sgd_impl.hpp | 49 ++++--- .../parallel_sgd/sparse_mc_function.hpp | 138 ------------------ .../parallel_sgd/sparse_mc_function_impl.hpp | 136 ----------------- 4 files changed, 30 insertions(+), 295 deletions(-) delete mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp delete mode 100644 src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp diff --git a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt index 9a62144ca5..48235b6506 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt +++ b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt @@ -1,8 +1,6 @@ set(SOURCES parallel_sgd.hpp parallel_sgd_impl.hpp - sparse_mc_function.hpp - sparse_mc_function_impl.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 5786ff56e6..e0c0058006 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -36,12 +36,39 @@ double ParallelSGD::Optimize( SparseFunctionType& function, arma::mat& iterate) { - double overallObjective = 0; - double lastObjective = DBL_MAX; + double overallObjective = DBL_MAX; + double lastObjective; - for (size_t i = 1; i != maxIterations; ++i){ + for (size_t i = 1; i != maxIterations; ++i) + { + // Calculate the overall objective. + lastObjective = overallObjective; overallObjective = 0; + for(size_t j = 0; j < function.NumFunctions(); ++j) + { + overallObjective += function.Evaluate(iterate, j); + } + + // Output current objective function. + Log::Info << "Parallel SGD: iteration " << i << ", objective " + << overallObjective << "." << std::endl; + + if (std::isnan(overallObjective) || std::isinf(overallObjective)) + { + Log::Warn << "Parallel SGD: converged to " << overallObjective + << "; terminating" << " with failure. Try a smaller step size?" + << std::endl; + return overallObjective; + } + + if (std::abs(lastObjective - overallObjective) < tolerance) + { + Log::Info << "SGD: minimized within tolerance " << tolerance << "; " + << "terminating optimization." << std::endl; + return overallObjective; + } + // Get the stepsize for this iteration double stepSize = decayPolicy.StepSize(i); @@ -77,22 +104,6 @@ double ParallelSGD::Optimize( } } } - - // Evaluate the function - overallObjective = 0; - for (size_t j = 0; j < function.NumFunctions(); ++j) - { - overallObjective += function.Evaluate(iterate, j); - } - - Log::Info << "\nObjective : " << overallObjective << " Iteration : " << i; - if (std::abs(overallObjective - lastObjective) < tolerance) - { - Log::Info << "\nParallel SGD terminated with objective delta " - << " within tolerance : " << overallObjective << std::endl; - return overallObjective; - } - lastObjective = overallObjective; } Log::Info << "\n Parallel SGD terminated with objective : " << overallObjective << std::endl; diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp deleted file mode 100644 index e13aa19c9c..0000000000 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function.hpp +++ /dev/null @@ -1,138 +0,0 @@ -/** - * @file sparse_mc_function.hpp - * @author Shikhar Bhardwaj - * - * Implementation of the hinge loss function for training a sparse SVM with the - * parallel SGD algorithm - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_HPP -#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_HPP -#include - -/** - * An implementation of the matrix completion example from HOGWILD!, based on - * empirical risk minimization in a sparse setting. - */ - -class SparseMCLossFunction{ - public: - //! Nothing to do for the default constructor. - SparseMCLossFunction() = default; - - /** - * Member initialization constructor. - * - * @param rows The row indices of the data points. - * @param cols The column indices of the data points. - * @param ratings The ratings of the data points. - * @param mu The regularization parameter. - * @param rank The width of the first factor. - */ - SparseMCLossFunction(const arma::uvec& rows, const arma::uvec& cols, - const arma::vec& ratings, double mu, size_t rank); - - /** - * Special initialization constructor. - * - * @param dataset The sparse matrix containing the datapoints. - * @param mu The regularization parameter. - * @param rank The width of the first factor. - */ - SparseMCLossFunction(const arma::sp_mat& dataset, double mu, size_t rank); - - /** - * Evaluate the squared error function with the given parameters at the id-th - * data point. - * - * @param weights The decision variable at which the function is to be - * evaluated. - * @param id Index of point to use for objective function evaluation. - * @return The value of the loss function at the given parameter. - */ - double Evaluate(const arma::mat& weights, size_t id); - - /** - * Evaluate the gradient of the squared error with the given parameters. - * - * @tparam GradType The type of the gradient parameter. - * @param weights The decision variable at which the gradient is to be - * evaluated. - * @param id Index of point to use for objective function evaluation. - * @param gradient Out param for the gradient. - */ - template - void Gradient(const arma::mat& weights, size_t id, GradType& gradient); - - /** - * Get the recovered matrix from the iterate. - * @param weights The decision variable at which the gradient is to be - * evaluated. - * @return The recovered matrix after completion. - */ - arma::mat Recover(const arma::mat& weights); - - //! Get the height of the sparse matrix. - size_t NumRows() const { return numRows; } - //! Modify the height of the sparse matrix. - size_t& NumRows() { return numRows; } - - //! Get the width of the sparse matrix. - size_t NumCols() const { return numCols; } - //! Modify the width of the sparse matrix. - size_t& NumCols() { return numCols; } - - //! Get the regularization parameter. - double Mu() const { return mu; } - //! Modify the regularization parameter. - double& Mu() { return mu; } - - //! Get the rank. - size_t Rank() const { return rank; } - //! Modify the rank. - size_t& Rank() { return rank; } - - //! Return the number of functions. - size_t NumFunctions(); - - private: - //! Calculate the frequency tables and mean rating before calling Evaluate - //! or Gradient. - void CalculateStatistics(); - - //! The row index of the datapoints. - arma::uvec rows; - - //! The column index of the datapoints. - arma::uvec cols; - - //! The rating of the datapoints. - arma::vec ratings; - - //! The frequency of the columns. - arma::uvec colCnt; - - //! The frequency of the rows. - arma::uvec rowCnt; - - //! The regularization parameter. - double mu; - - //! The height of the sparse matrix - size_t numRows; - - //! The width of the sparse matrix - size_t numCols; - - //! The width of the first factor. - size_t rank; -}; - -// Include implementation -#include "sparse_mc_function_impl.hpp" - -#endif diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp deleted file mode 100644 index 6c52120ab1..0000000000 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_mc_function_impl.hpp +++ /dev/null @@ -1,136 +0,0 @@ -/** - * @file sparse_mc_function_impl.hpp - * @author Shikhar Bhardwaj - * - * Implementation of the sparse matrix factorization example loss function. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_IMPL_HPP -#define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_SPARSE_MC_IMPL_HPP - -// In case it hasn't been included yet. -#include "sparse_mc_function.hpp" - -SparseMCLossFunction::SparseMCLossFunction(const arma::uvec& rows, - const arma::uvec& cols, - const arma::vec& ratings, - double mu, - size_t rank) : - rows(rows), cols(cols), ratings(ratings), mu(mu), rank(rank) -{ - CalculateStatistics(); -} - -SparseMCLossFunction::SparseMCLossFunction(const arma::sp_mat& dataset, - double mu, - size_t rank) : mu(mu), rank(rank) -{ - // Extract the relevant data from the sparse matrix representation. - std::vector instance_rows, instance_cols; - std::vector instance_ratings; - for (size_t i = 0; i < dataset.n_cols; ++i) - { - for (auto cur = dataset.begin_col(i); cur != dataset.end_col(i); ++cur) - { - instance_cols.push_back(i); - instance_rows.push_back(cur.row()); - instance_ratings.push_back(*cur); - } - } - // Store the data in the object state. - rows = arma::uvec(instance_rows); - cols = arma::uvec(instance_cols); - ratings = arma::vec(instance_ratings); - CalculateStatistics(); -} - -void SparseMCLossFunction::CalculateStatistics() -{ - // Take one pass over the data to aggregate statistics. - numCols = arma::max(cols) + 1; - numRows = arma::max(rows) + 1; - - colCnt = arma::uvec(numCols, arma::fill::zeros); - rowCnt = arma::uvec(numRows, arma::fill::zeros); - - // Go through the data and calculate the required frequencies. - for (size_t i = 0; i < rows.n_elem; ++i) - { - rowCnt(rows(i))++; - colCnt(cols(i))++; - } -} - -double SparseMCLossFunction::Evaluate(const arma::mat& weights, size_t id) -{ - // The decision variable is expected to be stored as follows. - // The first numRows columns have the first factor matrix, the next numCols - // columns have the second factor matrix. The decision variable matrix is - // thus of size rank x (numCols + numRows). - - size_t colId = numRows + cols(id); - size_t rowId = rows(id); - - double error = arma::dot(weights.col(rowId), weights.col(colId)) - - ratings(id); - double loss = error * error; - - // Add the regularisation term. - if (rowCnt(rows(id)) > 1) - { - double rowNorm = arma::norm(weights.col(rowId)); - loss += (mu * rowNorm * rowNorm) / (2 * (rowCnt(rows(id)) - 1)); - } - if (colCnt(cols(id)) > 1) - { - double colNorm = arma::norm(weights.col(colId)); - loss += (mu * colNorm * colNorm) / (2 * (colCnt(cols(id)) - 1)); - } - - return loss; -} - -template -void SparseMCLossFunction::Gradient(const arma::mat& weights, size_t id, - GradType& gradient) -{ - // Index of the column corresponding to the row and column of the current - // example in the decision variable. - size_t colId = numRows + cols(id); - size_t rowId = rows(id); - - gradient = arma::sp_mat(rank, numCols + numRows); - double error = arma::dot(weights.col(rowId), weights.col(colId)) - - ratings(id); - - // Calculate gradient for the first factor. - // Add the regularisation term. - if (rowCnt(rows(id)) > 1) - gradient.col(rowId) = (mu / (rowCnt(rows(id)) - 1)) * weights.col(rowId); - - gradient.col(rowId) += error * weights.col(colId); - - // Calculate gradient for the second factor. - // Add the regularisation term. - if (colCnt(cols(id)) > 1) - gradient.col(colId) = (mu / (colCnt(cols(id)) - 1)) * weights.col(colId); - - gradient.col(colId) += error * weights.col(rowId); -} - -arma::mat SparseMCLossFunction::Recover(const arma::mat& weights) -{ - return arma::trans(weights.cols(0, numRows - 1)) * weights.cols(numRows, - numRows + numCols - 1); -} - -size_t SparseMCLossFunction::NumFunctions() -{ - return rows.n_elem; -} - -#endif From 343a2bdd35a69eadf31ba47ea2f19ad78f1ddd95 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Fri, 14 Jul 2017 03:21:00 +0530 Subject: [PATCH 31/41] Minor style and documentation fixes --- src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp | 2 +- src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index c82cb34ade..128c156a5d 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -48,7 +48,7 @@ namespace optimization { * * The Gradient function interface is slightly changed from the * DecomposableFunctionType interface, it takes in a sparse matrix as the - * out-param for the gradient. As ParallelSGD is only expected to be relevant in + * out-param for the gradient, as ParallelSGD is only expected to be relevant in * situations where the computed gradient is sparse. * * @tparam DecayPolicyType Step size update policy used by parallel SGD diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index e0c0058006..23d0f309c3 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -45,7 +45,7 @@ double ParallelSGD::Optimize( lastObjective = overallObjective; overallObjective = 0; - for(size_t j = 0; j < function.NumFunctions(); ++j) + for (size_t j = 0; j < function.NumFunctions(); ++j) { overallObjective += function.Evaluate(iterate, j); } @@ -114,6 +114,7 @@ template void ParallelSGD::GenerateVisitationOrder( arma::Col& visitationOrder, size_t numFunctions) { + // Generate a random vector of function indices. visitationOrder = arma::shuffle(arma::linspace>(0, (numFunctions - 1), numFunctions)); } From 58055f4dbb6977a9019f251d6a875ef801e81f65 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Sat, 15 Jul 2017 20:04:47 +0530 Subject: [PATCH 32/41] Templatize gradient in Sparse SVM function --- src/mlpack/methods/sparse_svm/sparse_svm_function.hpp | 3 ++- src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp b/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp index fe17c0838f..69141b1f50 100644 --- a/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp +++ b/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp @@ -40,7 +40,8 @@ class SparseSVMFunction{ * @param id Index of the datapoint to use for the gradient evaluation. * @param gradient Sparse matrix to output the gradient into. */ - void Gradient(const arma::mat& parameters, size_t id, arma::sp_mat& gradient); + template + void Gradient(const arma::mat& parameters, size_t id, GradType& gradient); //! Return the initial point for the optimization. const arma::mat& InitialPoint() const { return initialPoint; } diff --git a/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp b/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp index 5150abdda4..ab825f30be 100644 --- a/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp +++ b/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp @@ -27,13 +27,14 @@ double SparseSVMFunction::Evaluate(const arma::mat& parameters, size_t id) return std::max(0.0, 1 - labels(id) * arma::dot(dataset.col(id), parameters)); } +template void SparseSVMFunction::Gradient( - const arma::mat& parameters, size_t id, arma::sp_mat& gradient) + const arma::mat& parameters, size_t id, GradType& gradient) { // Evaluate the gradient of the hinge loss function. double dot = 1 - labels(id) * arma::dot(parameters, dataset.col(id)); - gradient = (dot < 0) ? arma::sp_mat(parameters.n_rows, 1) : - (-1 * arma::sp_mat(dataset.col(id) * labels(id))); + gradient = (dot < 0) ? GradType(parameters.n_rows, 1) : + (-1 * GradType(dataset.col(id) * labels(id))); } size_t SparseSVMFunction::NumFunctions() From f673a4f06da05853718df689fb2ed22668d96705 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Tue, 18 Jul 2017 01:21:36 +0530 Subject: [PATCH 33/41] Add default parameters to parallel SGD implementation Minor style and comment fixes. --- .../decay_policies/constant_step.hpp | 14 +++++++++--- .../decay_policies/exponential_backoff.hpp | 11 +++++----- .../optimizers/parallel_sgd/parallel_sgd.hpp | 22 ++++++------------- .../parallel_sgd/parallel_sgd_impl.hpp | 21 +++++++----------- src/mlpack/tests/parallel_sgd_test.cpp | 4 ++-- 5 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp index 1c456a9cec..50951e6874 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp @@ -23,16 +23,24 @@ namespace optimization { class ConstantStep { public: - ConstantStep(double initalStep) : step(initalStep) { /* Nothing to do */ } + /** + * Member initialization constructor. + * + * The defaults here are not necessarily good for the given problem, so it is + * suggested that the values used be tailored to the task at hand. + * + * @param step The intial stepsize to use. + */ + ConstantStep(const double step = 0.01) : step(step) { /* Nothing to do */ } /** * This function is called in each iteration before the gradient update. * - * @param n_epoch The iteration number for which the stepsize is to be + * @param numEpoch The iteration number for which the stepsize is to be * calculated. * @return The step size for the current iteration. */ - double StepSize(size_t /* n_epoch */) + double StepSize(size_t /* numEpoch */) { return step; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index 4f8b1da01c..11ee31d391 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -39,12 +39,13 @@ class ExponentialBackoff { public: /** - * Construct the exponential backoff policy with the required parameters. + * Member initializer constructor to construct the exponential backoff policy + * with the required parameters. * * @param firstBackoffEpoch The number of updates to run before the first * stepsize backoff. * @param step The initial stepsize(gamma). - * @param beta The reduction factor. + * @param beta The reduction factor. This should be a value in range (0, 1). */ ExponentialBackoff(size_t firstBackoffEpoch, double step, double beta) : firstBackoffEpoch(firstBackoffEpoch), step(step), beta(beta) @@ -54,12 +55,12 @@ class ExponentialBackoff /** * Get the step size for the current gradient update. * - * @param n_epoch The iteration number of the current update. + * @param numEpoch The iteration number of the current update. * @return The stepsize for the current iteration. */ - double StepSize(size_t n_epoch) + double StepSize(size_t numEpoch) { - if (n_epoch >= cutoffEpoch) + if (numEpoch >= cutoffEpoch) { step *= beta; cutoffEpoch += firstBackoffEpoch / beta; diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 128c156a5d..59520d4ce8 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -13,6 +13,7 @@ #define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_HPP #include +#include "decay_policies/constant_step.hpp" namespace mlpack { namespace optimization { @@ -54,7 +55,7 @@ namespace optimization { * @tparam DecayPolicyType Step size update policy used by parallel SGD * to update the stepsize after each iteration. */ -template +template class ParallelSGD { public: @@ -63,6 +64,9 @@ class ParallelSGD * the given parameters. One iteration means one batch of datapoints processed * by each thread. * + * The defaults here are not necessarily good for the given problem, so it is + * suggested that the values used be tailored to the task at hand. + * * @param maxIterations Maximum number of iterations allowed. * @param threadShareSize Number of datapoints to be processed in one * iteration by each thread. @@ -71,8 +75,8 @@ class ParallelSGD */ ParallelSGD(const size_t maxIterations, const size_t threadShareSize, - const double tolerance, - const DecayPolicyType& decayPolicy); + const double tolerance = 1e-5, + const DecayPolicyType& decayPolicy = DecayPolicyType()); /** * Optimize the given function using the parallel SGD algorithm. The given @@ -111,18 +115,6 @@ class ParallelSGD DecayPolicyType& DecayPolicy() { return decayPolicy; } private: - /** - * Generate the indices to be visited by each thread before iteration. - * Generates a randomly shuffled vector of datapoint indices (range 0 to - * function.NumFunctions() - 1). - * - * @param visitationOrder Out param with the indices of the datapoints for the - * current iteration. - * @param numFunctions The number of separable functions in the objective. - */ - void GenerateVisitationOrder(arma::Col& visitationOrder, - size_t numFunctions); - /** * Get the share of datapoint indices to be updated by the thread with given * thread id. diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 23d0f309c3..e85c86ea68 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -39,6 +39,10 @@ double ParallelSGD::Optimize( double overallObjective = DBL_MAX; double lastObjective; + // The order in which the functions will be visited. + arma::Col visitationOrder = arma::linspace>(0, + (function.NumFunctions() - 1), function.NumFunctions()); + for (size_t i = 1; i != maxIterations; ++i) { // Calculate the overall objective. @@ -72,13 +76,13 @@ double ParallelSGD::Optimize( // Get the stepsize for this iteration double stepSize = decayPolicy.StepSize(i); - arma::Col visitationOrder; - GenerateVisitationOrder(visitationOrder, function.NumFunctions()); + // Shuffle for uniform sampling of functions by each thread. + visitationOrder = arma::shuffle(visitationOrder); #pragma omp parallel { // Each processor gets a subset of the instances. - // Each subset is of size batchSize. + // Each subset is of size threadShareSize. arma::Col instances = ThreadShare(omp_get_thread_num(), visitationOrder); for (size_t j = 0; j < instances.n_elem; ++j) @@ -110,15 +114,6 @@ double ParallelSGD::Optimize( return overallObjective; } -template -void ParallelSGD::GenerateVisitationOrder( - arma::Col& visitationOrder, size_t numFunctions) -{ - // Generate a random vector of function indices. - visitationOrder = arma::shuffle(arma::linspace>(0, - (numFunctions - 1), numFunctions)); -} - template arma::Col ParallelSGD::ThreadShare( size_t threadId, const arma::Col& visitationOrder) @@ -136,7 +131,7 @@ arma::Col ParallelSGD::ThreadShare( } else { - // Equal distribution of batchSize examples to each thread. + // Equal distribution of threadShareSize examples to each thread. return visitationOrder.subvec(threadId * threadShareSize, (threadId + 1) * threadShareSize - 1); } diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index 06043c8ebc..48bfb27dd9 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -108,8 +108,8 @@ BOOST_AUTO_TEST_CASE(ThreadSharingTest) ParallelSGD s(0, 4, 1e-10, decayPolicy); // Generate a random visitation order. - arma::Col visitationOrder; - s.GenerateVisitationOrder(visitationOrder, 10); + arma::Col visitationOrder = arma::linspace>(0, + 9, 10); // Lets count how many times each example is handed out in an iteration. arma::Col count(10, arma::fill::zeros); From d1edab7c04cbaf1c97e1147855801664740e7e9b Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Tue, 18 Jul 2017 01:30:15 +0530 Subject: [PATCH 34/41] Update and clarify comments in parallel SGD implementation. --- src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp | 3 ++- .../core/optimizers/parallel_sgd/parallel_sgd_impl.hpp | 7 +++++-- .../core/optimizers/parallel_sgd/sparse_test_function.hpp | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 59520d4ce8..404ae14e31 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -67,7 +67,8 @@ class ParallelSGD * The defaults here are not necessarily good for the given problem, so it is * suggested that the values used be tailored to the task at hand. * - * @param maxIterations Maximum number of iterations allowed. + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). * @param threadShareSize Number of datapoints to be processed in one * iteration by each thread. * @param tolerance Maximum absolute tolerance to terminate the algorithm. diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index e85c86ea68..bab2d3c824 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -43,6 +43,9 @@ double ParallelSGD::Optimize( arma::Col visitationOrder = arma::linspace>(0, (function.NumFunctions() - 1), function.NumFunctions()); + // Iterate till the objective is within tolerance or the maximum number of + // allowed iterations is reached. If maxIterations is 0, this will iterate + // till convergence. for (size_t i = 1; i != maxIterations; ++i) { // Calculate the overall objective. @@ -61,7 +64,7 @@ double ParallelSGD::Optimize( if (std::isnan(overallObjective) || std::isinf(overallObjective)) { Log::Warn << "Parallel SGD: converged to " << overallObjective - << "; terminating" << " with failure. Try a smaller step size?" + << "; terminating with failure. Try a smaller step size?" << std::endl; return overallObjective; } @@ -99,7 +102,7 @@ double ParallelSGD::Optimize( for (size_t i = 0; i < gradient.n_cols; ++i) { // Iterate over the non-zero elements. - for (auto cur = gradient.begin_col(i); cur != gradient.end_col(i); + for (arma::sp_mat::iterator cur = gradient.begin_col(i); cur != gradient.end_col(i); ++cur) { #pragma omp atomic diff --git a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp index fb48a49efc..1428fe49f0 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/sparse_test_function.hpp @@ -19,7 +19,7 @@ namespace optimization { namespace test { // A simple test function. Each dimension has a parabola with a -// distinct minima. Each update is guaranteed to be sparse(only a single +// distinct minimum. Each update is guaranteed to be sparse(only a single // dimension is updated in the decision variable by each thread). At the end of // a reasonable number of iterations, each value in the decision variable should // be at the vertex of the parabola in that dimension. From 3e07fefa274d895221b5e28618251fc669d516c1 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Tue, 18 Jul 2017 01:44:03 +0530 Subject: [PATCH 35/41] Add sparse test function to sources and fix style issue --- src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt | 2 ++ .../core/optimizers/parallel_sgd/parallel_sgd_impl.hpp | 4 ++-- src/mlpack/tests/regularized_svd_test.cpp | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt index 48235b6506..b1914cd3e6 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt +++ b/src/mlpack/core/optimizers/parallel_sgd/CMakeLists.txt @@ -1,6 +1,8 @@ set(SOURCES parallel_sgd.hpp parallel_sgd_impl.hpp + sparse_test_function.hpp + sparse_test_function_impl.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index bab2d3c824..a9dfb5853e 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -102,8 +102,8 @@ double ParallelSGD::Optimize( for (size_t i = 0; i < gradient.n_cols; ++i) { // Iterate over the non-zero elements. - for (arma::sp_mat::iterator cur = gradient.begin_col(i); cur != gradient.end_col(i); - ++cur) + for (arma::sp_mat::iterator cur = gradient.begin_col(i); + cur != gradient.end_col(i); ++cur) { #pragma omp atomic iterate(cur.row(), i) -= stepSize * (*cur); diff --git a/src/mlpack/tests/regularized_svd_test.cpp b/src/mlpack/tests/regularized_svd_test.cpp index 5c3fce6489..9b91daa2b3 100644 --- a/src/mlpack/tests/regularized_svd_test.cpp +++ b/src/mlpack/tests/regularized_svd_test.cpp @@ -286,8 +286,11 @@ BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionOptimizeHOGWILD) ConstantStep decayPolicy(alpha); + // Iterate till convergence. + // The threadShareSize is chosen such that each function gets optimized. ParallelSGD optimizer(0, - rSVDFunc.NumFunctions() / omp_get_max_threads(), 1e-5, decayPolicy); + std::ceil((float) rSVDFunc.NumFunctions() / omp_get_max_threads()), 1e-5, + decayPolicy); // Obtain optimized parameters after training. arma::mat optParameters = arma::randu(rank, numUsers + numItems); From 13052bff62d17064b7b37aad96a758663120bc45 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Fri, 21 Jul 2017 15:39:55 +0530 Subject: [PATCH 36/41] Add specialized overload for parallel SGD in Regularized SVD --- .../regularized_svd_function.hpp | 15 ++- .../regularized_svd_function_impl.hpp | 100 ++++++++++++++++++ 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp index 03e496d347..7465d8fa2d 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp @@ -15,6 +15,8 @@ #include #include +#include +#include namespace mlpack { namespace svd { @@ -128,9 +130,10 @@ namespace mlpack { namespace optimization { /** - * Template specialization for SGD optimizer. Used because the gradient - * affects only a small number of parameters per example, and thus the normal - * abstraction does not work as fast as we might like it to. + * Template specialization for the SGD and parallel SGD optimizer. Used + * because the gradient affects only a small number of parameters per example, + * and thus the normal abstraction does not work as fast as we might like it + * to. */ template <> template <> @@ -138,6 +141,12 @@ namespace optimization { mlpack::svd::RegularizedSVDFunction& function, arma::mat& parameters); + template <> + template <> + inline double ParallelSGD::Optimize( + mlpack::svd::RegularizedSVDFunction& function, + arma::mat& parameters); + } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp index 31200b681c..8f5ef63ad6 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp @@ -218,6 +218,106 @@ double StandardSGD::Optimize( return overallObjective; } + +template <> +template <> +inline double ParallelSGD::Optimize( + mlpack::svd::RegularizedSVDFunction& function, + arma::mat& iterate) +{ + double overallObjective = DBL_MAX; + double lastObjective; + + // The order in which the functions will be visited. + arma::Col visitationOrder = arma::linspace>(0, + (function.NumFunctions() - 1), function.NumFunctions()); + + const arma::mat data = function.Dataset(); + + // Iterate till the objective is within tolerance or the maximum number of + // allowed iterations is reached. If maxIterations is 0, this will iterate + // till convergence. + for (size_t i = 1; i != maxIterations; ++i) + { + // Calculate the overall objective. + lastObjective = overallObjective; + overallObjective = 0; + + #pragma omp parallel for reduction(+:overallObjective) + for (size_t j = 0; j < function.NumFunctions(); ++j) + { + overallObjective += function.Evaluate(iterate, j); + } + + // Output current objective function. + Log::Info << "Parallel SGD: iteration " << i << ", objective " + << overallObjective << "." << std::endl; + + if (std::isnan(overallObjective) || std::isinf(overallObjective)) + { + Log::Warn << "Parallel SGD: converged to " << overallObjective + << "; terminating with failure. Try a smaller step size?" + << std::endl; + return overallObjective; + } + + if (std::abs(lastObjective - overallObjective) < tolerance) + { + Log::Info << "SGD: minimized within tolerance " << tolerance << "; " + << "terminating optimization." << std::endl; + return overallObjective; + } + + // Get the stepsize for this iteration + double stepSize = decayPolicy.StepSize(i); + + // Shuffle for uniform sampling of functions by each thread. + std::random_shuffle(visitationOrder.begin(), visitationOrder.end()); + + #pragma omp parallel + { + // Each processor gets a subset of the instances. + // Each subset is of size threadShareSize. + arma::Col instances = ThreadShare(omp_get_thread_num(), + visitationOrder); + for (size_t j = 0; j < instances.n_elem; ++j) + { + const size_t numUsers = function.NumUsers(); + + // Indices for accessing the the correct parameter columns. + const size_t user = data(0, instances[j]); + const size_t item = data(1, instances[j]) + numUsers; + + // Prediction error for the example. + const double rating = data(2, instances[j]); + double ratingError = rating - arma::dot(iterate.col(user), + iterate.col(item)); + + double lambda = function.Lambda(); + + arma::mat userUpdate = stepSize * (lambda * iterate.col(user) - + ratingError * iterate.col(item)); + arma::mat itemUpdate = stepSize * (lambda * iterate.col(item) - + ratingError * iterate.col(user)); + + // Gradient is non-zero only for the parameter columns corresponding to + // the example. + for (size_t i = 0; i < iterate.n_rows; ++i) + { + #pragma omp atomic + iterate(i, user) -= userUpdate(i); + #pragma omp atomic + iterate(i, item) -= itemUpdate(i); + } + } + } + } + Log::Info << "\n Parallel SGD terminated with objective : " + << overallObjective << std::endl; + + return overallObjective; +} + } // namespace optimization } // namespace mlpack From 382e4b7cf709fe94ee3bc43937d5816420f04fb7 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Fri, 21 Jul 2017 15:53:14 +0530 Subject: [PATCH 37/41] Use const in constructor params and fix comments --- .../decay_policies/constant_step.hpp | 2 +- .../decay_policies/exponential_backoff.hpp | 16 ++++++++++------ .../regularized_svd/regularized_svd_function.hpp | 2 +- .../methods/sparse_svm/sparse_svm_function.hpp | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp index 50951e6874..d8ac6b39a6 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/constant_step.hpp @@ -40,7 +40,7 @@ class ConstantStep * calculated. * @return The step size for the current iteration. */ - double StepSize(size_t /* numEpoch */) + double StepSize(const size_t /* numEpoch */) { return step; } diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index 11ee31d391..ab77d09b63 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -47,18 +47,22 @@ class ExponentialBackoff * @param step The initial stepsize(gamma). * @param beta The reduction factor. This should be a value in range (0, 1). */ - ExponentialBackoff(size_t firstBackoffEpoch, double step, double beta) : - firstBackoffEpoch(firstBackoffEpoch), step(step), beta(beta) - { - cutoffEpoch = firstBackoffEpoch; - } + ExponentialBackoff(const size_t firstBackoffEpoch, + const double step, + const double beta) : + firstBackoffEpoch(firstBackoffEpoch), + cutoffEpoch(firstBackoffEpoch), + step(step), + beta(beta) + { /* Nothing to do. */ } + /** * Get the step size for the current gradient update. * * @param numEpoch The iteration number of the current update. * @return The stepsize for the current iteration. */ - double StepSize(size_t numEpoch) + double StepSize(const size_t numEpoch) { if (numEpoch >= cutoffEpoch) { diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp index 7465d8fa2d..2d4e840708 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp @@ -27,7 +27,7 @@ namespace svd { * * @tparam MatType The matrix type of the dataset. */ -template +template class RegularizedSVDFunction { public: diff --git a/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp b/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp index 69141b1f50..709550b39b 100644 --- a/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp +++ b/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp @@ -3,7 +3,7 @@ * @author Shikhar Bhardwaj * * Implementation of the hinge loss function for training a sparse SVM with the - * parallel SGD algorithm + * parallel SGD algorithm. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the From dbda61c30f936b9abbd5f192b3607d2e55ebe235 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Fri, 21 Jul 2017 18:21:18 +0530 Subject: [PATCH 38/41] Remove ThreadShare to avoid unnecessary copies and reduce memory usage Use omp_size_t in parallel for loop --- .../decay_policies/exponential_backoff.hpp | 8 ++--- .../optimizers/parallel_sgd/parallel_sgd.hpp | 12 ------- .../parallel_sgd/parallel_sgd_impl.hpp | 33 ++++------------- .../regularized_svd_function_impl.hpp | 16 +++++---- src/mlpack/tests/parallel_sgd_test.cpp | 36 ------------------- 5 files changed, 19 insertions(+), 86 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp index ab77d09b63..f159bb03ba 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/decay_policies/exponential_backoff.hpp @@ -50,10 +50,10 @@ class ExponentialBackoff ExponentialBackoff(const size_t firstBackoffEpoch, const double step, const double beta) : - firstBackoffEpoch(firstBackoffEpoch), - cutoffEpoch(firstBackoffEpoch), - step(step), - beta(beta) + firstBackoffEpoch(firstBackoffEpoch), + cutoffEpoch(firstBackoffEpoch), + step(step), + beta(beta) { /* Nothing to do. */ } /** diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 404ae14e31..2257a8ce5a 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -116,18 +116,6 @@ class ParallelSGD DecayPolicyType& DecayPolicy() { return decayPolicy; } private: - /** - * Get the share of datapoint indices to be updated by the thread with given - * thread id. - * - * @param thread_id The id of the current thread. Range 0-OMP_NUM_THREADS. - * @param visitationOrder The random list of datapoint indices for the current - * iteration. - * @return Vector of datapoint indices to be visited by the current thread. - */ - arma::Col ThreadShare(size_t threadId, - const arma::Col& visitationOrder); - //! The maximum number of allowed iterations. size_t maxIterations; diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index a9dfb5853e..f7280847ee 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -86,16 +86,18 @@ double ParallelSGD::Optimize( { // Each processor gets a subset of the instances. // Each subset is of size threadShareSize. - arma::Col instances = ThreadShare(omp_get_thread_num(), - visitationOrder); - for (size_t j = 0; j < instances.n_elem; ++j) + size_t threadId = omp_get_thread_num(); + + for (size_t j = threadId * threadShareSize; + j < (threadId + 1) * threadShareSize && j < visitationOrder.n_elem; + ++j) { // Each instance affects only some components of the decision variable. // So the gradient is sparse. arma::sp_mat gradient; // Evaluate the sparse gradient. - function.Gradient(iterate, instances[j], gradient); + function.Gradient(iterate, visitationOrder[j], gradient); // Update the decision variable with non-zero components of the // gradient. @@ -117,29 +119,6 @@ double ParallelSGD::Optimize( return overallObjective; } -template -arma::Col ParallelSGD::ThreadShare( - size_t threadId, const arma::Col& visitationOrder) -{ - if (threadId * threadShareSize >= visitationOrder.n_elem) - { - // No data for this thread. - return arma::Col(); - } - else if ((threadId + 1) * threadShareSize >= visitationOrder.n_elem) - { - // The last few elements. - return visitationOrder.subvec(threadId * threadShareSize , - visitationOrder.n_elem - 1); - } - else - { - // Equal distribution of threadShareSize examples to each thread. - return visitationOrder.subvec(threadId * threadShareSize, - (threadId + 1) * threadShareSize - 1); - } -} - } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp index 8f5ef63ad6..22ad50c4f2 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp @@ -244,7 +244,7 @@ inline double ParallelSGD::Optimize( overallObjective = 0; #pragma omp parallel for reduction(+:overallObjective) - for (size_t j = 0; j < function.NumFunctions(); ++j) + for (omp_size_t j = 0; j < (omp_size_t) function.NumFunctions(); ++j) { overallObjective += function.Evaluate(iterate, j); } @@ -278,18 +278,20 @@ inline double ParallelSGD::Optimize( { // Each processor gets a subset of the instances. // Each subset is of size threadShareSize. - arma::Col instances = ThreadShare(omp_get_thread_num(), - visitationOrder); - for (size_t j = 0; j < instances.n_elem; ++j) + size_t threadId = omp_get_thread_num(); + + for (size_t j = threadId * threadShareSize; + j < (threadId + 1) * threadShareSize && j < visitationOrder.n_elem; + ++j) { const size_t numUsers = function.NumUsers(); // Indices for accessing the the correct parameter columns. - const size_t user = data(0, instances[j]); - const size_t item = data(1, instances[j]) + numUsers; + const size_t user = data(0, visitationOrder[j]); + const size_t item = data(1, visitationOrder[j]) + numUsers; // Prediction error for the example. - const double rating = data(2, instances[j]); + const double rating = data(2, visitationOrder[j]); double ratingError = rating - arma::dot(iterate.col(user), iterate.col(item)); diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index 48bfb27dd9..e77390f33b 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -97,42 +97,6 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) } } -/** - * Test if the data points are divided correctly among the threads. - */ -BOOST_AUTO_TEST_CASE(ThreadSharingTest) -{ - ConstantStep decayPolicy(0); - - // Each thread gets a batch of size 4. - ParallelSGD s(0, 4, 1e-10, decayPolicy); - - // Generate a random visitation order. - arma::Col visitationOrder = arma::linspace>(0, - 9, 10); - - // Lets count how many times each example is handed out in an iteration. - arma::Col count(10, arma::fill::zeros); - - for (size_t threadId = 0; threadId < 4; ++threadId) - { - arma::Col share = s.ThreadShare(threadId, visitationOrder); - for (size_t i = 0; i < share.n_elem; ++i) - count(share(i))++; - - // The last thread to have some data. - if (threadId == 2) - BOOST_REQUIRE_EQUAL(share.n_elem, 2); - - // Only the first 3 threads get data. - if (threadId > 2) - BOOST_REQUIRE_EQUAL(share.n_elem, 0); - } - - // If everything is correct, each count should be 1 for each data point. - CheckMatrices(count, arma::Col(10, arma::fill::ones)); -} - /** * Test the correctness of the Exponential backoff stepsize decay policy. */ From 6fece6132d56647b054af5d076e3cfaa3d38eac5 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 24 Jul 2017 03:02:22 +0530 Subject: [PATCH 39/41] Add shuffle parameter to ParallelSGD --- .../core/optimizers/parallel_sgd/parallel_sgd.hpp | 12 ++++++++++++ .../optimizers/parallel_sgd/parallel_sgd_impl.hpp | 6 +++++- .../regularized_svd_function_impl.hpp | 4 ++-- src/mlpack/tests/parallel_sgd_test.cpp | 4 ++-- src/mlpack/tests/regularized_svd_test.cpp | 2 +- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 2257a8ce5a..3975a2d69d 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -72,11 +72,14 @@ class ParallelSGD * @param threadShareSize Number of datapoints to be processed in one * iteration by each thread. * @param tolerance Maximum absolute tolerance to terminate the algorithm. + * @param shuffle If true, the function order is shuffled; otherwise, each + * function is visited in linear order. * @param decayPolicy The step size update policy to use. */ ParallelSGD(const size_t maxIterations, const size_t threadShareSize, const double tolerance = 1e-5, + const bool shuffle = true, const DecayPolicyType& decayPolicy = DecayPolicyType()); /** @@ -110,6 +113,11 @@ class ParallelSGD //! Modify the tolerance for termination. double& Tolerance() { return tolerance; } + //! Get whether or not the individual functions are shuffled. + bool Shuffle() const { return shuffle; } + //! Modify whether or not the individual functions are shuffled. + bool& Shuffle() { return shuffle; } + //! Get the step size decay policy. DecayPolicyType& DecayPolicy() const { return decayPolicy; } //! Modify the step size decay policy. @@ -125,6 +133,10 @@ class ParallelSGD //! The tolerance for termination. double tolerance; + //! Controls whether or not the individual functions are shuffled when + //! iterating. + bool shuffle; + //! The step size decay policy. DecayPolicyType decayPolicy; }; diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index f7280847ee..3333f4cbc6 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -23,10 +23,12 @@ ParallelSGD::ParallelSGD( const size_t maxIterations, const size_t threadShareSize, const double tolerance, + const bool shuffle, const DecayPolicyType& decayPolicy) : maxIterations(maxIterations), threadShareSize(threadShareSize), tolerance(tolerance), + shuffle(shuffle), decayPolicy(decayPolicy) { /* Nothing to do. */ } @@ -80,7 +82,9 @@ double ParallelSGD::Optimize( double stepSize = decayPolicy.StepSize(i); // Shuffle for uniform sampling of functions by each thread. - visitationOrder = arma::shuffle(visitationOrder); + + if (shuffle) // Determine order of visitation. + visitationOrder = arma::shuffle(visitationOrder); #pragma omp parallel { diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp index 22ad50c4f2..b6746197c0 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp @@ -271,8 +271,8 @@ inline double ParallelSGD::Optimize( // Get the stepsize for this iteration double stepSize = decayPolicy.StepSize(i); - // Shuffle for uniform sampling of functions by each thread. - std::random_shuffle(visitationOrder.begin(), visitationOrder.end()); + if (shuffle) // Determine order of visitation. + visitationOrder = arma::shuffle(visitationOrder); #pragma omp parallel { diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp index e77390f33b..17746ad785 100644 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ b/src/mlpack/tests/parallel_sgd_test.cpp @@ -53,7 +53,7 @@ BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) size_t batchSize = std::ceil((float) f.NumFunctions() / i); - ParallelSGD s(10000, batchSize, 1e-5, decayPolicy); + ParallelSGD s(10000, batchSize, 1e-5, true, decayPolicy); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(f, coordinates); @@ -84,7 +84,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) ConstantStep decayPolicy(0.001); - ParallelSGD s(0, f.NumFunctions(), 1e-12, decayPolicy); + ParallelSGD s(0, f.NumFunctions(), 1e-12, true, decayPolicy); arma::mat coordinates = f.GetInitialPoint(); diff --git a/src/mlpack/tests/regularized_svd_test.cpp b/src/mlpack/tests/regularized_svd_test.cpp index 9b91daa2b3..88ff52654d 100644 --- a/src/mlpack/tests/regularized_svd_test.cpp +++ b/src/mlpack/tests/regularized_svd_test.cpp @@ -290,7 +290,7 @@ BOOST_AUTO_TEST_CASE(RegularizedSVDFunctionOptimizeHOGWILD) // The threadShareSize is chosen such that each function gets optimized. ParallelSGD optimizer(0, std::ceil((float) rSVDFunc.NumFunctions() / omp_get_max_threads()), 1e-5, - decayPolicy); + true, decayPolicy); // Obtain optimized parameters after training. arma::mat optParameters = arma::randu(rank, numUsers + numItems); From 25282ca43f29c370c024f03b3dd3b16e30f22abe Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 24 Jul 2017 03:22:52 +0530 Subject: [PATCH 40/41] Use std::shuffle instead of arma::shuffle in ParallelSGD --- .../core/optimizers/parallel_sgd/parallel_sgd_impl.hpp | 6 +++++- .../regularized_svd/regularized_svd_function_impl.hpp | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 3333f4cbc6..2081556bc1 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -45,6 +45,10 @@ double ParallelSGD::Optimize( arma::Col visitationOrder = arma::linspace>(0, (function.NumFunctions() - 1), function.NumFunctions()); + // A random number generator instance to be used for shuffling the order of + // visitation. + std::mt19937 gen{ std::random_device()() }; + // Iterate till the objective is within tolerance or the maximum number of // allowed iterations is reached. If maxIterations is 0, this will iterate // till convergence. @@ -84,7 +88,7 @@ double ParallelSGD::Optimize( // Shuffle for uniform sampling of functions by each thread. if (shuffle) // Determine order of visitation. - visitationOrder = arma::shuffle(visitationOrder); + std::shuffle(visitationOrder.begin(), visitationOrder.end(), gen); #pragma omp parallel { diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp index b6746197c0..2b790045fc 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp @@ -132,7 +132,7 @@ void RegularizedSVDFunction::Gradient(const arma::mat& parameters, template template -void RegularizedSVDFunction::Gradient(const arma::mat ¶meters, +void RegularizedSVDFunction::Gradient(const arma::mat& parameters, size_t id, GradType &gradient) const { @@ -232,6 +232,10 @@ inline double ParallelSGD::Optimize( arma::Col visitationOrder = arma::linspace>(0, (function.NumFunctions() - 1), function.NumFunctions()); + // A random number generator instance to be used for shuffling the order of + // visitation. + std::mt19937 gen{ std::random_device()() }; + const arma::mat data = function.Dataset(); // Iterate till the objective is within tolerance or the maximum number of @@ -272,7 +276,7 @@ inline double ParallelSGD::Optimize( double stepSize = decayPolicy.StepSize(i); if (shuffle) // Determine order of visitation. - visitationOrder = arma::shuffle(visitationOrder); + std::shuffle(visitationOrder.begin(), visitationOrder.end(), gen); #pragma omp parallel { From c60f3af3aabc252a5b576da73f28330b0bbc8309 Mon Sep 17 00:00:00 2001 From: Shikhar Bhardwaj Date: Mon, 24 Jul 2017 14:03:22 +0530 Subject: [PATCH 41/41] Use existing random number generator for std::shuffle --- src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp | 1 + .../core/optimizers/parallel_sgd/parallel_sgd_impl.hpp | 7 ++----- .../regularized_svd/regularized_svd_function_impl.hpp | 7 ++----- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp index 3975a2d69d..7e789b8473 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd.hpp @@ -13,6 +13,7 @@ #define MLPACK_CORE_OPTIMIZERS_PARALLEL_SGD_HPP #include +#include #include "decay_policies/constant_step.hpp" namespace mlpack { diff --git a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp index 2081556bc1..97cc630942 100644 --- a/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/parallel_sgd/parallel_sgd_impl.hpp @@ -45,10 +45,6 @@ double ParallelSGD::Optimize( arma::Col visitationOrder = arma::linspace>(0, (function.NumFunctions() - 1), function.NumFunctions()); - // A random number generator instance to be used for shuffling the order of - // visitation. - std::mt19937 gen{ std::random_device()() }; - // Iterate till the objective is within tolerance or the maximum number of // allowed iterations is reached. If maxIterations is 0, this will iterate // till convergence. @@ -88,7 +84,8 @@ double ParallelSGD::Optimize( // Shuffle for uniform sampling of functions by each thread. if (shuffle) // Determine order of visitation. - std::shuffle(visitationOrder.begin(), visitationOrder.end(), gen); + std::shuffle(visitationOrder.begin(), visitationOrder.end(), + mlpack::math::randGen); #pragma omp parallel { diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp index 2b790045fc..7a46959049 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function_impl.hpp @@ -232,10 +232,6 @@ inline double ParallelSGD::Optimize( arma::Col visitationOrder = arma::linspace>(0, (function.NumFunctions() - 1), function.NumFunctions()); - // A random number generator instance to be used for shuffling the order of - // visitation. - std::mt19937 gen{ std::random_device()() }; - const arma::mat data = function.Dataset(); // Iterate till the objective is within tolerance or the maximum number of @@ -276,7 +272,8 @@ inline double ParallelSGD::Optimize( double stepSize = decayPolicy.StepSize(i); if (shuffle) // Determine order of visitation. - std::shuffle(visitationOrder.begin(), visitationOrder.end(), gen); + std::shuffle(visitationOrder.begin(), visitationOrder.end(), + mlpack::math::randGen); #pragma omp parallel {