From baca5db3d417d518da02dbad6dc6d045dfc61d63 Mon Sep 17 00:00:00 2001 From: Konstantin Sidorov Date: Fri, 21 Jul 2017 19:46:48 +0300 Subject: [PATCH] Adding all optimization features not directly related to the Augmented RNNs GSoC project --- .../minibatch_sgd/minibatch_sgd.hpp | 23 +++- .../minibatch_sgd/minibatch_sgd_impl.hpp | 14 ++- src/mlpack/core/optimizers/sgd/sgd.hpp | 59 ++++++++++- src/mlpack/core/optimizers/sgd/sgd_impl.hpp | 29 ++++- src/mlpack/methods/ann/layer/CMakeLists.txt | 2 + .../methods/ann/layer/cross_entropy_error.hpp | 100 ++++++++++++++++++ .../ann/layer/cross_entropy_error_impl.hpp | 58 ++++++++++ src/mlpack/methods/ann/layer/layer_types.hpp | 2 + 8 files changed, 271 insertions(+), 16 deletions(-) create mode 100644 src/mlpack/methods/ann/layer/cross_entropy_error.hpp create mode 100644 src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp index 248bc90dae..3bcaf038d6 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp @@ -105,6 +105,8 @@ class MiniBatchSGDType * @param updatePolicy Instantiated update policy used to adjust the given * parameters. * @param decayPolicy Instantiated decay policy used to adjust the step size. + * @param resetPolicy Flag that determines whether update policy parameters + * are reset before every Optimize call. */ MiniBatchSGDType(const size_t batchSize = 1000, const double stepSize = 0.01, @@ -112,7 +114,8 @@ class MiniBatchSGDType const double tolerance = 1e-5, const bool shuffle = true, const UpdatePolicyType& updatePolicy = UpdatePolicyType(), - const DecayPolicyType& decayPolicy = DecayPolicyType()); + const DecayPolicyType& decayPolicy = DecayPolicyType(), + const bool resetPolicy = true); /** * Optimize the given function using mini-batch SGD. The given starting point @@ -122,10 +125,13 @@ class MiniBatchSGDType * @tparam DecomposableFunctionType Type of the function to be optimized. * @param function Function to optimize. * @param iterate Starting point (will be modified). + * @param resetPolicy Flag indicating whether update policy + * should be reset before running optimization. * @return Objective value of the final point. */ template - double Optimize(DecomposableFunctionType& function, arma::mat& iterate); + double Optimize(DecomposableFunctionType& function, + arma::mat& iterate); //! Get the batch size. size_t BatchSize() const { return batchSize; } @@ -152,6 +158,13 @@ class MiniBatchSGDType //! Modify whether or not the individual functions are shuffled. bool& Shuffle() { return shuffle; } + //! Get whether or not the update policy parameters + //! are reset before Optimize call. + bool ResetPolicy() const { return resetPolicy; } + //! Modify whether or not the update policy parameters + //! are reset before Optimize call. + bool& ResetPolicy() { return resetPolicy; } + //! Get the update policy. UpdatePolicyType UpdatePolicy() const { return updatePolicy; } //! Modify the update policy. @@ -184,6 +197,10 @@ class MiniBatchSGDType //! The decay policy used to update the parameters in each iteration. DecayPolicyType decayPolicy; + + //! Flag that determines whether update policy parameters + //! are reset before every Optimize call. + bool resetPolicy; }; using MiniBatchSGD = MiniBatchSGDType; @@ -194,4 +211,4 @@ using MiniBatchSGD = MiniBatchSGDType; // Include implementation. #include "minibatch_sgd_impl.hpp" -#endif +#endif \ No newline at end of file diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp index 776182cb87..59de823128 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp @@ -31,14 +31,16 @@ MiniBatchSGDType< const double tolerance, const bool shuffle, const UpdatePolicyType& updatePolicy, - const DecayPolicyType& decayPolicy) : + const DecayPolicyType& decayPolicy, + const bool resetPolicy) : batchSize(batchSize), stepSize(stepSize), maxIterations(maxIterations), tolerance(tolerance), shuffle(shuffle), updatePolicy(updatePolicy), - decayPolicy(decayPolicy) + decayPolicy(decayPolicy), + resetPolicy(resetPolicy) { /* Nothing to do. */ } //! Optimize the function (minimize). @@ -50,7 +52,8 @@ template double MiniBatchSGDType< UpdatePolicyType, DecayPolicyType ->::Optimize(DecomposableFunctionType& function, arma::mat& iterate) +>::Optimize(DecomposableFunctionType& function, + arma::mat& iterate) { // Find the number of functions. const size_t numFunctions = function.NumFunctions(); @@ -75,7 +78,8 @@ double MiniBatchSGDType< overallObjective += function.Evaluate(iterate, i); // Initialize the update policy. - updatePolicy.Initialize(iterate.n_rows, iterate.n_cols); + if (resetPolicy) + updatePolicy.Initialize(iterate.n_rows, iterate.n_cols); // Now iterate! arma::mat gradient(iterate.n_rows, iterate.n_cols); @@ -178,4 +182,4 @@ double MiniBatchSGDType< } // namespace optimization } // namespace mlpack -#endif +#endif \ No newline at end of file diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index 4801df158a..bc2ce88481 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -92,12 +92,26 @@ class SGD * @param tolerance Maximum absolute tolerance to terminate algorithm. * @param shuffle If true, the function order is shuffled; otherwise, each * function is visited in linear order. + * @param updatePolicy Instantiated update policy used to adjust the given + * parameters. + * @param resetPolicy Flag that determines whether update policy parameters + * are reset before every Optimize call. + * @param clipGradient Flag that determines whether gradient should be + * clipped to some range before every SGD step. + * @param minGradient Minimum gradient value + * (affects optimization iff clipGradient flag is on). + * @param maxGradient Maximum gradient value + * (affects optimization iff clipGradient flag is on). */ SGD(const double stepSize = 0.01, const size_t maxIterations = 100000, const double tolerance = 1e-5, const bool shuffle = true, - const UpdatePolicyType updatePolicy = UpdatePolicyType()); + const UpdatePolicyType updatePolicy = UpdatePolicyType(), + const bool resetPolicy = true, + const bool clipGradient = false, + const double minGradient = 0.0, + const double maxGradient = 0.0); /** * Optimize the given function using stochastic gradient descent. The given @@ -110,7 +124,8 @@ class SGD * @return Objective value of the final point. */ template - double Optimize(DecomposableFunctionType& function, arma::mat& iterate); + double Optimize(DecomposableFunctionType& function, + arma::mat& iterate); //! Get the step size. double StepSize() const { return stepSize; } @@ -132,8 +147,30 @@ class SGD //! Modify whether or not the individual functions are shuffled. bool& Shuffle() { return shuffle; } + //! Get whether or not the update policy parameters + //! are reset before Optimize call. + bool ResetPolicy() const { return resetPolicy; } + //! Modify whether or not the update policy parameters + //! are reset before Optimize call. + bool& ResetPolicy() { return resetPolicy; } + + //! Get whether or not the gradient is clipped. + bool ClipGradient() const { return clipGradient; } + //! Modify whether or not the gradient is clipped. + bool& ClipGradient() { return clipGradient; } + + //! Get minimum gradient value. + double MinGradient() const { return minGradient; } + //! Modify minimum gradient value. + double& MinGradient() { return minGradient; } + + //! Get maximum gradient value. + double MaxGradient() const { return maxGradient; } + //! Modify maximum gradient value. + double& MaxGradient() { return maxGradient; } + //! Get the update policy. - const UpdatePolicyType& UpdatePolicy() const { return updatePolicy; } + UpdatePolicyType UpdatePolicy() const { return updatePolicy; } //! Modify the update policy. UpdatePolicyType& UpdatePolicy() { return updatePolicy; } @@ -153,6 +190,20 @@ class SGD //! The update policy used to update the parameters in each iteration. UpdatePolicyType updatePolicy; + + //! Flag indicating whether update policy + //! should be reset before running optimization. + bool resetPolicy; + + //! Flag that determines whether gradient should be clipped + //! to some range before every SGD step. + bool clipGradient; + + //! Minimum gradient value. + double minGradient; + + //! Maximum gradient value. + double maxGradient; }; using StandardSGD = SGD; @@ -165,4 +216,4 @@ using MomentumSGD = SGD; // Include implementation. #include "sgd_impl.hpp" -#endif +#endif \ No newline at end of file diff --git a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp index 224b60e445..5486f47081 100644 --- a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp @@ -29,12 +29,20 @@ SGD::SGD( const size_t maxIterations, const double tolerance, const bool shuffle, - const UpdatePolicyType updatePolicy) : + const UpdatePolicyType updatePolicy, + const bool resetPolicy, + const bool clipGradient, + const double minGradient, + const double maxGradient) : stepSize(stepSize), maxIterations(maxIterations), tolerance(tolerance), shuffle(shuffle), - updatePolicy(updatePolicy) + updatePolicy(updatePolicy), + resetPolicy(resetPolicy), + clipGradient(clipGradient), + minGradient(minGradient), + maxGradient(maxGradient) { /* Nothing to do. */ } //! Optimize the function (minimize). @@ -65,7 +73,8 @@ double SGD::Optimize( overallObjective += function.Evaluate(iterate, i); // Initialize the update policy. - updatePolicy.Initialize(iterate.n_rows, iterate.n_cols); + if (resetPolicy) + updatePolicy.Initialize(iterate.n_rows, iterate.n_cols); // Now iterate! arma::mat gradient(iterate.n_rows, iterate.n_cols); @@ -107,6 +116,18 @@ double SGD::Optimize( else function.Gradient(iterate, currentFunction, gradient); + // Clip the gradient. + if (clipGradient) + { + gradient.transform + ( + [&](double val) + { + return std::min(std::max(val, minGradient), maxGradient); + } + ); + } + // Use the update policy to take a step. updatePolicy.Update(iterate, stepSize, gradient); @@ -135,4 +156,4 @@ double SGD::Optimize( } // namespace optimization } // namespace mlpack -#endif +#endif \ No newline at end of file diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 17f54d6991..caa1cf112c 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -14,6 +14,8 @@ set(SOURCES constant_impl.hpp convolution.hpp convolution_impl.hpp + cross_entropy_error.hpp + cross_entropy_error_impl.hpp dropconnect.hpp dropconnect_impl.hpp dropout.hpp diff --git a/src/mlpack/methods/ann/layer/cross_entropy_error.hpp b/src/mlpack/methods/ann/layer/cross_entropy_error.hpp new file mode 100644 index 0000000000..086b731af9 --- /dev/null +++ b/src/mlpack/methods/ann/layer/cross_entropy_error.hpp @@ -0,0 +1,100 @@ +/** + * @file cross_entropy_error.hpp + * @author Konstantin Sidorov + * + * Definition of the cross-entropy performance function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_CROSS_ENTROPY_ERROR_HPP +#define MLPACK_METHODS_ANN_LAYER_CROSS_ENTROPY_ERROR_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The cross-entropy performance function measures the network's + * performance according to the cross-entropy + * between the input and target distributions. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class CrossEntropyError +{ + public: + /** + * Create the CrossEntropyError object. + */ + CrossEntropyError(); + + /* + * Computes the cross-entropy function. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + double Forward(const arma::Mat&& input, const arma::Mat&& target); + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated input activation. + * @param target The target vector. + * @param output The calculated error. + */ + template + void Backward(const arma::Mat&& input, + const arma::Mat&& target, + arma::Mat&& output); + + //! Get the input parameter. + InputDataType& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + /** + * Serialize the layer + */ + template + void Serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class CrossEntropyError + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "cross_entropy_error_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp new file mode 100644 index 0000000000..82c95e8f04 --- /dev/null +++ b/src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp @@ -0,0 +1,58 @@ +/** + * @file cross_entropy_error_impl.hpp + * @author Konstantin Sidorov + * + * Implementation of the cross-entropy performance function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_CROSS_ENTROPY_ERROR_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_CROSS_ENTROPY_ERROR_IMPL_HPP + +// In case it hasn't yet been included. +#include "cross_entropy_error.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +CrossEntropyError::CrossEntropyError() +{ + // Nothing to do here. +} + +template +template +double CrossEntropyError::Forward( + const arma::Mat&& input, const arma::Mat&& target) +{ + return -arma::accu(target % arma::trunc_log(input) + + (1. - target) % arma::trunc_log(1. - input)); +} + +template +template +void CrossEntropyError::Backward( + const arma::Mat&& input, + const arma::Mat&& target, + arma::Mat&& output) +{ + output = (1. - target) / (1. - input + 1e-2) - target / (input + 1e-2); +} + +template +template +void CrossEntropyError::Serialize( + Archive& /* ar */, + const unsigned int /* version */) +{ + // Nothing to do here. +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 708838bc01..82a1662e57 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -33,6 +33,7 @@ #include #include #include +#include // Convolution modules. #include @@ -89,6 +90,7 @@ using LayerTypes = boost::variant< Convolution, NaiveConvolution, NaiveConvolution, arma::mat, arma::mat>*, + CrossEntropyError*, DropConnect*, Dropout*, ELU*,