Adding all optimization features not directly related to the Augmented RNNs GSoC project

src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp

@@ -105,6 +105,8 @@ class MiniBatchSGDType
    * @param updatePolicy Instantiated update policy used to adjust the given
    *     parameters.
    * @param decayPolicy Instantiated decay policy used to adjust the step size.
+   * @param resetPolicy Flag that determines whether update policy parameters
+   *                    are reset before every Optimize call.
    */
   MiniBatchSGDType(const size_t batchSize = 1000,
                    const double stepSize = 0.01,
@@ -112,7 +114,8 @@ class MiniBatchSGDType
                    const double tolerance = 1e-5,
                    const bool shuffle = true,
                    const UpdatePolicyType& updatePolicy = UpdatePolicyType(),
-                   const DecayPolicyType& decayPolicy = DecayPolicyType());
+                   const DecayPolicyType& decayPolicy = DecayPolicyType(),
+                   const bool resetPolicy = true);
 
   /**
    * Optimize the given function using mini-batch SGD.  The given starting point
@@ -122,10 +125,13 @@ class MiniBatchSGDType
    * @tparam DecomposableFunctionType Type of the function to be optimized.
    * @param function Function to optimize.
    * @param iterate Starting point (will be modified).
+   * @param resetPolicy Flag indicating whether update policy
+   *                   should be reset before running optimization.
    * @return Objective value of the final point.
    */
   template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  double Optimize(DecomposableFunctionType& function,
+                  arma::mat& iterate);
 
   //! Get the batch size.
   size_t BatchSize() const { return batchSize; }
@@ -152,6 +158,13 @@ class MiniBatchSGDType
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return shuffle; }
 
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return resetPolicy; }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return resetPolicy; } 
+
   //! Get the update policy.
   UpdatePolicyType UpdatePolicy() const { return updatePolicy; }
   //! Modify the update policy.
@@ -184,6 +197,10 @@ class MiniBatchSGDType
 
   //! The decay policy used to update the parameters in each iteration.
   DecayPolicyType decayPolicy;
+
+  //! Flag that determines whether update policy parameters
+  //! are reset before every Optimize call.
+  bool resetPolicy;
 };
 
 using MiniBatchSGD = MiniBatchSGDType<VanillaUpdate, NoDecay>;
@@ -194,4 +211,4 @@ using MiniBatchSGD = MiniBatchSGDType<VanillaUpdate, NoDecay>;
 // Include implementation.
 #include "minibatch_sgd_impl.hpp"
 
-#endif
+#endif

src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp

@@ -31,14 +31,16 @@ MiniBatchSGDType<
                     const double tolerance,
                     const bool shuffle,
                     const UpdatePolicyType& updatePolicy,
-                    const DecayPolicyType& decayPolicy) :
+                    const DecayPolicyType& decayPolicy,
+                    const bool resetPolicy) :
                     batchSize(batchSize),
                     stepSize(stepSize),
                     maxIterations(maxIterations),
                     tolerance(tolerance),
                     shuffle(shuffle),
                     updatePolicy(updatePolicy),
-                    decayPolicy(decayPolicy)
+                    decayPolicy(decayPolicy),
+                    resetPolicy(resetPolicy)
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
@@ -50,7 +52,8 @@ template<typename DecomposableFunctionType>
 double MiniBatchSGDType<
     UpdatePolicyType,
     DecayPolicyType
->::Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+>::Optimize(DecomposableFunctionType& function,
+            arma::mat& iterate)
 {
   // Find the number of functions.
   const size_t numFunctions = function.NumFunctions();
@@ -75,7 +78,8 @@ double MiniBatchSGDType<
     overallObjective += function.Evaluate(iterate, i);
 
   // Initialize the update policy.
-  updatePolicy.Initialize(iterate.n_rows, iterate.n_cols);
+  if (resetPolicy)
+    updatePolicy.Initialize(iterate.n_rows, iterate.n_cols);
 
   // Now iterate!
   arma::mat gradient(iterate.n_rows, iterate.n_cols);
@@ -178,4 +182,4 @@ double MiniBatchSGDType<
 } // namespace optimization
 } // namespace mlpack
 
-#endif
+#endif

src/mlpack/core/optimizers/sgd/sgd.hpp

@@ -92,12 +92,26 @@ class SGD
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param shuffle If true, the function order is shuffled; otherwise, each
    *     function is visited in linear order.
+   * @param updatePolicy Instantiated update policy used to adjust the given
+   *                     parameters.
+   * @param resetPolicy Flag that determines whether update policy parameters
+   *                    are reset before every Optimize call.
+   * @param clipGradient Flag that determines whether gradient should be
+   *                     clipped to some range before every SGD step.
+   * @param minGradient Minimum gradient value
+   *                    (affects optimization iff clipGradient flag is on).
+   * @param maxGradient Maximum gradient value
+   *                    (affects optimization iff clipGradient flag is on).
    */
   SGD(const double stepSize = 0.01,
       const size_t maxIterations = 100000,
       const double tolerance = 1e-5,
       const bool shuffle = true,
-      const UpdatePolicyType updatePolicy = UpdatePolicyType());
+      const UpdatePolicyType updatePolicy = UpdatePolicyType(),
+      const bool resetPolicy = true,
+      const bool clipGradient = false,
+      const double minGradient = 0.0,
+      const double maxGradient = 0.0);
 
   /**
    * Optimize the given function using stochastic gradient descent.  The given
@@ -110,7 +124,8 @@ class SGD
    * @return Objective value of the final point.
    */
   template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);
+  double Optimize(DecomposableFunctionType& function,
+                  arma::mat& iterate);
 
   //! Get the step size.
   double StepSize() const { return stepSize; }
@@ -132,8 +147,30 @@ class SGD
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return shuffle; }
 
+  //! Get whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool ResetPolicy() const { return resetPolicy; }
+  //! Modify whether or not the update policy parameters
+  //! are reset before Optimize call.
+  bool& ResetPolicy() { return resetPolicy; }
+
+  //! Get whether or not the gradient is clipped.
+  bool ClipGradient() const { return clipGradient; }
+  //! Modify whether or not the gradient is clipped.
+  bool& ClipGradient() { return clipGradient; }
+
+  //! Get minimum gradient value.
+  double MinGradient() const { return minGradient; }
+  //! Modify minimum gradient value.
+  double& MinGradient() { return minGradient; }
+
+  //! Get maximum gradient value.
+  double MaxGradient() const { return maxGradient; }
+  //! Modify maximum gradient value.
+  double& MaxGradient() { return maxGradient; }
+
   //! Get the update policy.
-  const UpdatePolicyType& UpdatePolicy() const { return updatePolicy; }
+  UpdatePolicyType UpdatePolicy() const { return updatePolicy; }
   //! Modify the update policy.
   UpdatePolicyType& UpdatePolicy() { return updatePolicy; }
 
@@ -153,6 +190,20 @@ class SGD
 
   //! The update policy used to update the parameters in each iteration.
   UpdatePolicyType updatePolicy;
+
+  //! Flag indicating whether update policy
+  //! should be reset before running optimization.
+  bool resetPolicy;
+
+  //! Flag that determines whether gradient should be clipped 
+  //! to some range before every SGD step.
+  bool clipGradient;
+
+  //! Minimum gradient value.
+  double minGradient;
+
+  //! Maximum gradient value.
+  double maxGradient;
 };
 
 using StandardSGD = SGD<VanillaUpdate>;
@@ -165,4 +216,4 @@ using MomentumSGD = SGD<MomentumUpdate>;
 // Include implementation.
 #include "sgd_impl.hpp"
 
-#endif
+#endif

src/mlpack/core/optimizers/sgd/sgd_impl.hpp

@@ -29,12 +29,20 @@ SGD<UpdatePolicyType>::SGD(
     const size_t maxIterations,
     const double tolerance,
     const bool shuffle,
-    const UpdatePolicyType updatePolicy) :
+    const UpdatePolicyType updatePolicy,
+    const bool resetPolicy,
+    const bool clipGradient,
+    const double minGradient,
+    const double maxGradient) :
     stepSize(stepSize),
     maxIterations(maxIterations),
     tolerance(tolerance),
     shuffle(shuffle),
-    updatePolicy(updatePolicy)
+    updatePolicy(updatePolicy),
+    resetPolicy(resetPolicy),
+    clipGradient(clipGradient),
+    minGradient(minGradient),
+    maxGradient(maxGradient)
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
@@ -65,7 +73,8 @@ double SGD<UpdatePolicyType>::Optimize(
     overallObjective += function.Evaluate(iterate, i);
 
   // Initialize the update policy.
-  updatePolicy.Initialize(iterate.n_rows, iterate.n_cols);
+  if (resetPolicy)
+    updatePolicy.Initialize(iterate.n_rows, iterate.n_cols);
 
   // Now iterate!
   arma::mat gradient(iterate.n_rows, iterate.n_cols);
@@ -107,6 +116,18 @@ double SGD<UpdatePolicyType>::Optimize(
     else
       function.Gradient(iterate, currentFunction, gradient);
 
+    // Clip the gradient.
+    if (clipGradient)
+    {
+      gradient.transform
+      (
+        [&](double val)
+        {
+          return std::min(std::max(val, minGradient), maxGradient);
+        }
+      );
+    }
+
     // Use the update policy to take a step.
     updatePolicy.Update(iterate, stepSize, gradient);
 
@@ -135,4 +156,4 @@ double SGD<UpdatePolicyType>::Optimize(
 } // namespace optimization
 } // namespace mlpack
 
-#endif
+#endif

src/mlpack/methods/ann/layer/CMakeLists.txt

@@ -14,6 +14,8 @@ set(SOURCES
   constant_impl.hpp
   convolution.hpp
   convolution_impl.hpp
+  cross_entropy_error.hpp
+  cross_entropy_error_impl.hpp
   dropconnect.hpp
   dropconnect_impl.hpp
   dropout.hpp

src/mlpack/methods/ann/layer/cross_entropy_error.hpp

@@ -0,0 +1,100 @@
+/**
+ * @file cross_entropy_error.hpp
+ * @author Konstantin Sidorov
+ *
+ * Definition of the cross-entropy performance function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_CROSS_ENTROPY_ERROR_HPP
+#define MLPACK_METHODS_ANN_LAYER_CROSS_ENTROPY_ERROR_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The cross-entropy performance function measures the network's
+ * performance according to the cross-entropy
+ * between the input and target distributions.
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class CrossEntropyError
+{
+ public:
+  /**
+   * Create the CrossEntropyError object.
+   */
+  CrossEntropyError();
+
+  /*
+   * Computes the cross-entropy function.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename eT>
+  double Forward(const arma::Mat<eT>&& input, const arma::Mat<eT>&& target);
+  /**
+   * Ordinary feed backward pass of a neural network.
+   *
+   * @param input The propagated input activation.
+   * @param target The target vector.
+   * @param output The calculated error.
+   */
+  template<typename eT>
+  void Backward(const arma::Mat<eT>&& input,
+                const arma::Mat<eT>&& target,
+                arma::Mat<eT>&& output);
+
+  //! Get the input parameter.
+  InputDataType& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  OutputDataType& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  /**
+   * Serialize the layer
+   */
+  template<typename Archive>
+  void Serialize(Archive& ar, const unsigned int /* version */);
+
+ private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+}; // class CrossEntropyError
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "cross_entropy_error_impl.hpp"
+
+#endif

src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp

@@ -0,0 +1,58 @@
+/**
+ * @file cross_entropy_error_impl.hpp
+ * @author Konstantin Sidorov
+ *
+ * Implementation of the cross-entropy performance function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_CROSS_ENTROPY_ERROR_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_CROSS_ENTROPY_ERROR_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "cross_entropy_error.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+CrossEntropyError<InputDataType, OutputDataType>::CrossEntropyError()
+{
+  // Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+double CrossEntropyError<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>&& input, const arma::Mat<eT>&& target)
+{
+  return -arma::accu(target % arma::trunc_log(input) +
+                     (1. - target) % arma::trunc_log(1. - input));
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void CrossEntropyError<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>&& input,
+    const arma::Mat<eT>&& target,
+    arma::Mat<eT>&& output)
+{
+  output = (1. - target) / (1. - input + 1e-2) - target / (input + 1e-2);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void CrossEntropyError<InputDataType, OutputDataType>::Serialize(
+    Archive& /* ar */,
+    const unsigned int /* version */)
+{
+  // Nothing to do here.
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/layer_types.hpp

@@ -33,6 +33,7 @@
 #include <mlpack/methods/ann/layer/parametric_relu.hpp>
 #include <mlpack/methods/ann/layer/reinforce_normal.hpp>
 #include <mlpack/methods/ann/layer/select.hpp>
+#include <mlpack/methods/ann/layer/cross_entropy_error.hpp>
 
 // Convolution modules.
 #include <mlpack/methods/ann/convolution_rules/border_modes.hpp>
@@ -89,6 +90,7 @@ using LayerTypes = boost::variant<
     Convolution<NaiveConvolution<ValidConvolution>,
                 NaiveConvolution<FullConvolution>,
                 NaiveConvolution<ValidConvolution>, arma::mat, arma::mat>*,
+    CrossEntropyError<arma::mat, arma::mat>*,
     DropConnect<arma::mat, arma::mat>*,
     Dropout<arma::mat, arma::mat>*,
     ELU<arma::mat, arma::mat>*,