Merge branch 'master' into master

src/mlpack/core.hpp

@@ -242,6 +242,7 @@
  *   - Moksh Jain <mokshjn00@gmail.com>
  *   - Manthan-R-Sheth <manthanrsheth96@gmail.com>
  *	 - Namrata Mukhija <namratamukhija@gmail.com>
+ *   - Rohan Raj <rajrohan1108@gmail.com>
  */
 
 // First, include all of the prerequisites.

src/mlpack/methods/ann/init_rules/CMakeLists.txt

@@ -3,8 +3,10 @@
 set(SOURCES
   const_init.hpp
   gaussian_init.hpp
+  he_init.hpp
   init_rules_traits.hpp
   kathirvalavakumar_subavathi_init.hpp
+  lecun_normal_init.hpp
   network_init.hpp
   nguyen_widrow_init.hpp
   oivs_init.hpp

src/mlpack/methods/ann/init_rules/he_init.hpp

@@ -0,0 +1,106 @@
+/**
+ * @file he_init.hpp
+ * @author Dakshit Agrawal
+ * @author Prabhat Sharma
+ *
+ * Intialization rule given by He et. al. for neural networks. The He
+ * initialization initializes weights of the neural network to better
+ * suit the rectified activation units.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP
+#define MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/core/math/random.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * This class is used to initialize weight matrix with the He
+ * initialization rule given by He et. al. for neural networks. The He
+ * initialization initializes weights of the neural network to better
+ * suit the rectified activation units.
+ *
+ * For more information, the following paper can be referred to:
+ *
+ * @code
+ * @article{Delving2015,
+ *   title   = {Delving Deep into Rectifiers: Surpassing Human-Level Performance
+ *              on ImageNet Classification},
+ *   author  = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
+ *   journal = {2015 IEEE International Conference on Computer Vision (ICCV)},
+ *   year    = {2015},
+ *   pages   = {1026-1034}
+ * }
+ * @endcode
+ *
+ */
+class HeInitialization
+{
+ public:
+  /**
+   * Initialize the HeInitialization object.
+   */
+  HeInitialization()
+  {
+    // Nothing to do here.
+  }
+
+  /**
+   * Initialize the elements of the weight matrix with the He initialization
+   * rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   */
+  void Initialize(arma::mat& W, const size_t rows, const size_t cols)
+  {
+    // He initialization rule says to initialize weights with random
+    // values taken from a gaussian distribution with mean = 0 and
+    // standard deviation = sqrt(2/rows), i.e. variance = (2/rows).
+    const double variance = 2.0 / (double)rows;
+
+    if (W.is_empty())
+    {
+      W.set_size(rows, cols);
+    }
+
+    // Multipling a random variable X with variance V(X) by some factor c,
+    // then the variance V(cX) = (c^2) * V(X).
+    W.imbue( [&]() { return sqrt(variance) * arma::randn(); } );
+  }
+
+  /**
+   * Initialize the elements of the specified weight 3rd order tensor
+   * with He initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   * @param slice Numbers of slices.
+   */
+  void Initialize(arma::cube & W,
+                  const size_t rows,
+                  const size_t cols,
+                  const size_t slices)
+  {
+    if (W.is_empty())
+      W.set_size(rows, cols, slices);
+
+    for (size_t i = 0; i < slices; i++)
+      Initialize(W.slice(i), rows, cols);
+  }
+}; // class HeInitialization
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp

@@ -0,0 +1,112 @@
+/**
+ * @file lecun_normal_init.hpp
+ * @author Dakshit Agrawal
+ * @author Prabhat Sharma
+ *
+ * Intialization rule given by Lecun et. al. for neural networks and
+ * also mentioned in Self Normalizing Networks.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP
+#define MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/core/math/random.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * This class is used to initialize weight matrix with the Lecun Normalization
+ * initialization rule.
+ *
+ * For more information, the following papers can be referred to:
+ *
+ * @code
+ * @inproceedings{Klambauer2017,
+ *   itle  = {Self-Normalizing Neural Networks.},
+ *   author = {Klambauer, Günter and Unterthiner, Thomas
+ *             and Mayr, Andreas and Hochreiter, Sepp},
+ *   pages  = {972-981},
+ *   year   = {2017}
+ * }
+ *
+ * @inproceedings{LeCun1998,
+ *   title  = {Efficient BackProp},
+ *   author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B.
+ *             and M\"{u}ller, Klaus-Robert},
+ *   year   = {1998},
+ *   pages  = {9--50}
+ * }
+ * @endcode
+ *
+*/
+class LecunNormalInitialization
+{
+ public:
+  /**
+   * Initialize the LecunNormalInitialization object.
+   */
+  LecunNormalInitialization()
+  {
+    // Nothing to do here.
+  }
+
+  /**
+   * Initialize the elements of the weight matrix with the Lecun
+   * Normal initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   */
+  void Initialize(arma::mat& W,
+                  const size_t rows,
+                  const size_t cols)
+  {
+    // He initialization rule says to initialize weights with random
+    // values taken from a gaussian distribution with mean = 0 and
+    // standard deviation = sqrt(1 / rows), i.e. variance = (1 / rows).
+    const double variance = 1.0 / ((double) rows);
+
+    if (W.is_empty())
+    {
+      W.set_size(rows, cols);
+    }
+
+    // Multipling a random variable X with variance V(X) by some factor c,
+    // then the variance V(cX) = (c ^ 2) * V(X).
+    W.imbue( [&]() { return sqrt(variance) * arma::randn(); } );
+  }
+
+  /**
+   * Initialize the elements of the specified weight 3rd order tensor
+   * with Lecun Normal initialization rule.
+   *
+   * @param W Weight matrix to initialize.
+   * @param rows Number of rows.
+   * @param cols Number of columns.
+   * @param slice Numbers of slices.
+   */
+  void Initialize(arma::cube & W,
+                  const size_t rows,
+                  const size_t cols,
+                  const size_t slices)
+  {
+    if (W.is_empty())
+      W.set_size(rows, cols, slices);
+
+    for (size_t i = 0; i < slices; i++)
+      Initialize(W.slice(i), rows, cols);
+  }
+}; // class LecunNormalInitialization
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/CMakeLists.txt

@@ -30,6 +30,8 @@ set(SOURCES
   elu_impl.hpp
   fast_lstm.hpp
   fast_lstm_impl.hpp
+  flexible_relu.hpp
+  flexible_relu_impl.hpp
   glimpse.hpp
   glimpse_impl.hpp
   gru.hpp

src/mlpack/methods/ann/layer/dropout.hpp

@@ -3,7 +3,7 @@
  * @author Marcus Edel
  *
  * Definition of the Dropout class, which implements a regularizer that
- * randomly sets units to zero. Preventing units from co-adapting.
+ * randomly sets units to zero preventing units from co-adapting.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -18,12 +18,12 @@
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
+
 /**
- * The dropout layer is a regularizer that randomly with probability ratio
+ * The dropout layer is a regularizer that randomly with probability 'ratio'
  * sets input values to zero and scales the remaining elements by factor 1 /
- * (1 - ratio). If rescale is true the input is scaled with 1 / (1-p) when
- * deterministic is false. In the deterministic mode (during testing), the layer
- * just scales the output.
+ * (1 - ratio) rather than during test time so as to keep the expected sum same.
+ * In the deterministic mode (during testing), there is no change in the input.
  *
  * Note: During training you should set deterministic to false and during
  * testing you should set deterministic to true.
@@ -47,21 +47,17 @@ namespace ann /** Artificial Neural Network. */ {
  * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
  */
-template <
-    typename InputDataType = arma::mat,
-    typename OutputDataType = arma::mat
->
+template<typename InputDataType = arma::mat,
+         typename OutputDataType = arma::mat>
 class Dropout
 {
  public:
   /**
-   * Create the Dropout object using the specified ratio and rescale
-   * parameter.
+   * Create the Dropout object using the specified ratio parameter.
    *
    * @param ratio The probability of setting a value to zero.
-   * @param rescale If true the input is rescaled when deterministic is False.
    */
-  Dropout(const double ratio = 0.5, const bool rescale = true);
+  Dropout(const double ratio = 0.5);
 
   /**
    * Ordinary feed forward pass of the dropout layer.
@@ -85,7 +81,7 @@ class Dropout
                 arma::Mat<eT>&& g);
 
   //! Get the input parameter.
-  InputDataType const& InputParameter() const { return inputParameter; }
+  InputDataType const& InputParameter() const { return inputParameter;  }
   //! Modify the input parameter.
   InputDataType& InputParameter() { return inputParameter; }
 
@@ -114,11 +110,6 @@ class Dropout
     scale = 1.0 / (1.0 - ratio);
   }
 
-  //! The value of the rescale parameter.
-  bool Rescale() const {return rescale; }
-  //! Modify the value of the rescale parameter.
-  bool& Rescale() {return rescale; }
-
   /**
    * Serialize the layer.
    */
@@ -146,9 +137,6 @@ class Dropout
 
   //! If true dropout and scaling is disabled, see notes above.
   bool deterministic;
-
-  //! If true the input is rescaled when deterministic is False.
-  bool rescale;
 }; // class Dropout
 
 } // namespace ann

src/mlpack/methods/ann/layer/dropout_impl.hpp

@@ -21,11 +21,10 @@ namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputDataType, typename OutputDataType>
 Dropout<InputDataType, OutputDataType>::Dropout(
-    const double ratio, const bool rescale) :
+    const double ratio) :
     ratio(ratio),
     scale(1.0 / (1.0 - ratio)),
-    deterministic(true),
-    rescale(rescale)
+    deterministic(false)
 {
   // Nothing to do here.
 }
@@ -40,21 +39,14 @@ void Dropout<InputDataType, OutputDataType>::Forward(
   // (during testing).
   if (deterministic)
   {
-    if (!rescale)
-    {
-      output = input;
-    }
-    else
-    {
-      output = input * scale;
-    }
+    output = input;
   }
   else
   {
     // Scale with input / (1 - ratio) and set values to zero with probability
-    // ratio.
+    // 'ratio'.
     mask = arma::randu<arma::Mat<eT> >(input.n_rows, input.n_cols);
-    mask.transform( [&](double val) { return (val > ratio); } );
+    mask.transform([&](double val) { return (val > ratio); });
     output = input % mask * scale;
   }
 }
@@ -76,7 +68,6 @@ void Dropout<InputDataType, OutputDataType>::serialize(
     const unsigned int /* version */)
 {
   ar & BOOST_SERIALIZATION_NVP(ratio);
-  ar & BOOST_SERIALIZATION_NVP(rescale);
 
   // Reset scale.
   scale = 1.0 / (1.0 - ratio);

src/mlpack/methods/ann/layer/flexible_relu.hpp

@@ -0,0 +1,175 @@
+/**
+ * @file flexible_relu.hpp
+ * @author Aarush Gupta
+ * @author Manthan-R-Sheth
+ *
+ * Definition of FlexibleReLU layer as described by
+ * Suo Qiu, Xiangmin Xu and Bolun Cai in
+ * "FReLU: Flexible Rectified Linear Units for Improving Convolutional
+ * Neural Networks", 2018
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_HPP
+#define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /**Artificial Neural Network*/ {
+
+/**
+ * The FlexibleReLU activation function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) &=& \max(0,x)+alpha \\
+ * f'(x) &=& \left\{
+ * 	 \begin(array){lr}
+ * 	   1 & : x > 0 \\
+ * 	   0 & : x \le 0
+ * 	 \end{array}
+ * \right
+ * @f}
+ *
+ * For more information, read the following paper:
+ *
+ * @code
+ * @article{Qiu2018,
+ *  author  = {Suo Qiu, Xiangmin Xu and Bolun Cai},
+ *  title   = {FReLU: Flexible Rectified Linear Units for Improving
+ *             Convolutional Neural Networks}
+ *  journal = {arxiv preprint},
+ *  URL     = {https://arxiv.org/abs/1706.08098},
+ *  year    = {2018}
+ * }
+ * @endcode
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mar,
+ *         arma::sp_mat or arma::cube)
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube)
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class FlexibleReLU
+{
+ public:
+  /**
+   *
+   * Create the FlexibleReLU object using the specified parameters.
+   * The non zero parameter can be adjusted by specifying the parameter
+   * alpha which controls the range of the relu function. (Default alpha = 0)
+   * This parameter is trainable.
+   *
+   * @param alpha Parameter for adjusting the range of the relu function.
+   *
+   */
+  FlexibleReLU(const double alpha = 0);
+
+  /**
+   * Reset the layer parameter.
+   */
+  void Reset();
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename InputType, typename OutputType>
+  void Forward(const InputType&& input, OutputType&& output);
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename DataType>
+  void Backward(const DataType&& input, DataType&& gy, DataType&& g);
+
+  /**
+   * Calculate the gradient using the output delta and the input activation.
+   *
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
+   */
+  template<typename eT>
+  void Gradient(const arma::Mat<eT>&& input,
+                arma::Mat<eT>&& error,
+                arma::Mat<eT>&& gradient);
+
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return alpha; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return alpha; }
+
+  //! Get the input parameter.
+  InputDataType const& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta;}
+
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
+  //! Get the parameter controlling the range of the relu function.
+  double const& Alpha() const { return alpha; }
+  //! Modify the parameter controlling the range of the relu function.
+  double& Alpha() { return alpha; }
+
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version*/);
+
+ private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Parameter object.
+  OutputDataType alpha;
+
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
+  //! Parameter controlling the range of the rectifier function
+  double userAlpha;
+}; // class FlexibleReLU
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation
+#include "flexible_relu_impl.hpp"
+
+#endif

src/mlpack/methods/ann/layer/flexible_relu_impl.hpp

@@ -0,0 +1,86 @@
+/**
+ * @file flexible_relu_impl.hpp
+ * @author Aarush Gupta
+ * @author Manthan-R-Sheth
+ *
+ * Implementation of FlexibleReLU layer as described by
+ * Suo Qiu, Xiangmin Xu and Bolun Cai in
+ * "FReLU: Flexible Rectified Linear Units for Improving Convolutional
+ *  Neural Networks", 2018
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP
+
+#include "flexible_relu.hpp"
+#include<algorithm>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+FlexibleReLU<InputDataType, OutputDataType>::FlexibleReLU(
+    const double alpha) : userAlpha(alpha)
+{
+  this->alpha.set_size(1, 1);
+  this->alpha(0) = userAlpha;
+}
+
+template<typename InputDataType, typename OutputDataType>
+void FlexibleReLU<InputDataType, OutputDataType>::Reset()
+{
+  //! Set value of alpha to the one given by user.
+  alpha(0) = userAlpha;
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename OutputType>
+void FlexibleReLU<InputDataType, OutputDataType>::Forward(
+    const InputType&& input, OutputType&& output)
+{
+  output = arma::clamp(input, 0.0, DBL_MAX) + alpha(0);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename DataType>
+void FlexibleReLU<InputDataType, OutputDataType>::Backward(
+    const DataType&& input, DataType&& gy, DataType&& g)
+{
+  //! Compute the first derivative of FlexibleReLU function.
+  g = gy % arma::clamp(arma::sign(input), 0.0, 1.0);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void FlexibleReLU<InputDataType, OutputDataType>::Gradient(
+    const arma::Mat<eT>&& input,
+    arma::Mat<eT>&& error,
+    arma::Mat<eT>&& gradient)
+{
+  if (gradient.n_elem == 0)
+  {
+    gradient.set_size(1, 1);
+  }
+
+  gradient(0) = arma::accu(error) / input.n_cols;
+}
+
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void FlexibleReLU<InputDataType, OutputDataType>::serialize(
+    Archive& ar,
+    const unsigned int /* version*/)
+{
+  ar & BOOST_SERIALIZATION_NVP(alpha);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/layer_types.hpp

@@ -27,6 +27,7 @@
 #include <mlpack/methods/ann/layer/hard_tanh.hpp>
 #include <mlpack/methods/ann/layer/join.hpp>
 #include <mlpack/methods/ann/layer/leaky_relu.hpp>
+#include <mlpack/methods/ann/layer/flexible_relu.hpp>
 #include <mlpack/methods/ann/layer/log_softmax.hpp>
 #include <mlpack/methods/ann/layer/lookup.hpp>
 #include <mlpack/methods/ann/layer/mean_squared_error.hpp>
@@ -126,6 +127,7 @@ using LayerTypes = boost::variant<
     Dropout<arma::mat, arma::mat>*,
     AlphaDropout<arma::mat, arma::mat>*,
     ELU<arma::mat, arma::mat>*,
+    FlexibleReLU<arma::mat, arma::mat>*,
     Glimpse<arma::mat, arma::mat>*,
     HardTanH<arma::mat, arma::mat>*,
     Join<arma::mat, arma::mat>*,

src/mlpack/methods/ann/layer/leaky_relu.hpp

@@ -103,7 +103,7 @@ class LeakyReLU
 
  private:
   /**
-   * Computes the LeakReLU function
+   * Computes the LeakyReLU function
    *
    * @param x Input data.
    * @return f(x).
@@ -114,7 +114,7 @@ class LeakyReLU
   }
 
   /**
-   * Computes the Leaky ReLU function using a dense matrix as input.
+   * Computes the LeakyReLU function using a dense matrix as input.
    *
    * @param x Input data.
    * @param y The resulting output activation.

src/mlpack/methods/ann/rnn_impl.hpp

@@ -201,6 +201,7 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
   ResetCells();
 
   double performance = 0;
+  size_t responseSeq = 0;
 
   for (size_t seqNum = 0; seqNum < rho; ++seqNum)
   {
@@ -208,8 +209,10 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
     arma::mat stepData(predictors.slice(seqNum).colptr(begin),
         predictors.n_rows, batchSize, false, true);
     Forward(std::move(stepData));
-    arma::mat respData(responses.slice(seqNum).colptr(begin),
-        responses.n_rows, batchSize, false, true);
+    if (!single)
+    {
+      responseSeq = seqNum;
+    }
 
     if (!deterministic)
     {
@@ -222,7 +225,7 @@ double RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Evaluate(
 
     performance += outputLayer.Forward(std::move(boost::apply_visitor(
         outputParameterVisitor, network.back())),
-        std::move(arma::mat(responses.slice(seqNum).colptr(begin),
+        std::move(arma::mat(responses.slice(responseSeq).colptr(begin),
             responses.n_rows, batchSize, false, true)));
   }
 
@@ -283,19 +286,25 @@ void RNN<OutputLayerType, InitializationRuleType, CustomLayers...>::Gradient(
     {
       error.zeros();
     }
+    else if (single && seqNum == 0)
+    {
+      outputLayer.Backward(std::move(boost::apply_visitor(
+          outputParameterVisitor, network.back())),
+          std::move(arma::mat(responses.slice(0).colptr(begin),
+          responses.n_rows, batchSize, false, true)), std::move(error));
+    }
     else
     {
       outputLayer.Backward(std::move(boost::apply_visitor(
           outputParameterVisitor, network.back())),
           std::move(arma::mat(responses.slice(rho - seqNum - 1).colptr(begin),
-              responses.n_rows, batchSize, false, true)),
-          std::move(error));
+          responses.n_rows, batchSize, false, true)), std::move(error));
     }
 
     Backward();
     Gradient(std::move(
         arma::mat(predictors.slice(rho - seqNum - 1).colptr(begin),
-            predictors.n_rows, batchSize, false, true)));
+        predictors.n_rows, batchSize, false, true)));
     gradient += currentGradient;
   }
 }

src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt

@@ -3,6 +3,7 @@
 set(SOURCES
   mountain_car.hpp
   cart_pole.hpp
+  acrobat.hpp
 )
 
 # Add directory name to sources.

src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp

@@ -0,0 +1,345 @@
+/**
+ * @file acrobat.hpp
+ * @author Rohan Raj
+ *
+ * This file is an implementation of Acrobat task:
+ * https://gym.openai.com/envs/Acrobot-v1/
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_RL_ENVIRONMENT_ACROBAT_HPP
+#define MLPACK_METHODS_RL_ENVIRONMENT_ACROBAT_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack{
+namespace rl{
+
+/**
+ * Implementation of Acrobat game. Acrobot is a 2-link pendulum with only the
+ * second joint actuated. Intitially, both links point downwards. The goal is
+ * to swing the end-effector at a height at least the length of one link above
+ * the base. Both links can swing freely and can pass by each other, i.e.,
+ * they don't collide when they have the same angle.
+ */
+class Acrobat
+{
+ public:
+   /*
+    * Implementation of Acrobat State. Each State is a tuple vector
+    * (theta1, thetha2, angular velocity 1, angular velocity 2).
+    */
+  class State
+  {
+   public:
+    /**
+     * Construct a state instance.
+     */
+    State(): data(dimension) { /* nothing to do here */ }
+
+    /**
+     * Construct a state instance from given data.
+     *
+     * @param data Data for the theta and angular velocity of two links.
+     */
+    State(const arma::colvec& data) : data(data)
+    { /* nothing to do here */ }
+
+    //! Modify the state representation.
+    arma::colvec& Data() { return data; }
+
+    //! Get value of theta (one).
+    double Theta1() const { return data[0]; }
+    //! Modify value of theta (one).
+    double& Theta1() { return data[0]; }
+
+    //! Get value of theta (two).
+    double Theta2() const { return data[1]; }
+    //! Modify value of theta (two).
+    double& Theta2() { return data[1]; }
+
+    //! Get value of Angular velocity (one).
+    double AngularVelocity1() const { return data[2]; }
+    //! Modify the angular velocity (one).
+    double& AngularVelocity1() { return data[2]; }
+
+    //! Get value of Angular velocity (two).
+    double AngularVelocity2() const { return data[3]; }
+    //! Modify the angular velocity (two).
+    double& AngularVelocity2() { return data[3]; }
+
+    //! Encode the state to a column vector.
+    const arma::colvec& Encode() const { return data; }
+
+    //! Dimension of the encoded state.
+    static constexpr size_t dimension = 4;
+
+   private:
+    //! Locally-Stored (theta1, theta2, angular velocity 1, angular velocity2).
+    arma::colvec data;
+  };
+
+  /*
+   * Implementation of action for Acrobat
+   */
+  enum Action
+  {
+    negativeTorque,
+    zeroTorque,
+    positiveTorque,
+
+    // Track the size of the action space.
+    size
+  };
+
+   /**
+    * Construct a Acrobat instance using the given constants.
+    *
+    * @param gravity The gravity parameter.
+    * @param linkLength1 The length of link 1.
+    * @param linkLength2 The length of link 2.
+    * @param linkMass1 The mass of link 1.
+    * @param linkMass2 The mass of link 2.
+    * @param linkCom1 The position of the center of mass of link 1.
+    * @param linkCom2 The position of the center of mass of link 2.
+    * @param linkMoi The moments of inertia for both link.
+    * @param maxVel1 The max angular velocity of link1.
+    * @param maxVel2 The max angular velocity of link2.
+    * @param dt The differential value.
+    */
+  Acrobat(const double gravity = 9.81,
+          const double linkLength1 = 1.0,
+          const double linkLength2 = 1.0,
+          const double linkMass1 = 1.0,
+          const double linkMass2 = 1.0,
+          const double linkCom1 = 0.5,
+          const double linkCom2 = 0.5,
+          const double linkMoi = 1.0,
+          const double maxVel1 = 4 * M_PI,
+          const double maxVel2 = 9 * M_PI,
+          const double dt = 0.2) :
+      gravity(gravity),
+      linkLength1(linkLength1),
+      linkLength2(linkLength2),
+      linkMass1(linkMass1),
+      linkMass2(linkMass2),
+      linkCom1(linkCom1),
+      linkCom2(linkCom2),
+      linkMoi(linkMoi),
+      maxVel1(maxVel1),
+      maxVel2(maxVel2),
+      dt(dt)
+  { /* Nothing to do here */ }
+
+  /**
+   * Dynamics of the Acrobat System. To get reward and next state based on
+   * current state and current action. Always return -1 reward.
+   *
+   * @param state The current State.
+   * @param action The action taken.
+   * @param nextState The next state.
+   * @return reward, it's always -1.0.
+   */
+  double Sample(const State& state,
+                const Action& action,
+                State& nextState) const
+  {
+    // Make a vector to estimate nextstate.
+    arma::colvec currentState = {state.Theta1(), state.Theta2(),
+        state.AngularVelocity1(), state.AngularVelocity2()};
+
+    arma::colvec currentNextState = Rk4(currentState, Torque(action));
+
+    nextState.Theta1() = Wrap(currentNextState[0], -M_PI, M_PI);
+
+    nextState.Theta2() = Wrap(currentNextState[1], -M_PI, M_PI);
+    //! The value of angular velocity is bounded in min and max value.
+    nextState.AngularVelocity1() = std::min(
+        std::max(currentNextState[2], -maxVel1), maxVel1);
+    nextState.AngularVelocity2() = std::min(
+        std::max(currentNextState[3], -maxVel2), maxVel2);
+
+    return -1.0;
+  };
+
+  /**
+   * Dynamics of the Acrobat System. To get reward and next state based on
+   * current state and current action. This function calls the Sample function
+   * to estimate the next state return reward for taking a particular action.
+   *
+   * @param state The current State.
+   * @param action The action taken.
+   * @param nextState The next state.
+   */
+  double Sample(const State& state, const Action& action) const
+  {
+    State nextState;
+    return Sample(state, action, nextState);
+  }
+
+  /**
+   * This function does random initialization of state space.
+   */
+  State InitialSample() const
+  {
+    return State((arma::randu<arma::colvec>(4) - 0.5) / 5.0);
+  }
+
+  /**
+   * This function checks if the acrobat has reached the terminal state.
+   *
+   * @param state The current State.
+   */
+  bool IsTerminal(const State& state) const
+  {
+    return bool (-std::cos(state.Theta1())-std::cos(state.Theta1() +
+        state.Theta2()) > 1.0);
+  }
+
+  /**
+   * This is the ordinary differential equations required for estimation of
+   * nextState through RK4 method.
+   *
+   * @param state Current State.
+   * @param torque The torque Applied.
+   */
+  arma::colvec Dsdt(arma::colvec state, const double torque) const
+  {
+    const double m1 = linkMass1;
+    const double m2 = linkMass2;
+    const double l1 = linkLength1;
+    const double lc1 = linkCom1;
+    const double lc2 = linkCom2;
+    const double I1 = linkMoi;
+    const double I2 = linkMoi;
+    const double g = gravity;
+    const double a = torque;
+    const double theta1 = state[0];
+    const double theta2 = state[1];
+
+    arma::colvec values(4);
+    values[0] = state[2];
+    values[1] = state[3];
+
+    const double d1 = m1 * std::pow(lc1, 2) + m2 * (std::pow(l1, 2) +
+        std::pow(lc2, 2) + 2 * l1 * lc2 * std::cos(theta2)) + I1 + I2;
+
+    const double d2 = m2 * (std::pow(lc2, 2) + l1 * lc2 * std::cos(theta2)) +
+        I2;
+
+    const double phi2 = m2 * lc2 * g * std::cos(theta1 + theta2 - M_PI / 2.);
+
+    const double phi1 = - m2 * l1 * lc2 * std::pow(values[1], 2) *
+        std::sin(theta2) - 2 * m2 * l1 * lc2 * values[1] * values[0] *
+        std::sin(theta2) + (m1 * lc1 +  m2 * l1) * g *
+        std::cos(theta1 - M_PI / 2) + phi2;
+
+    values[3] = (a + d2 / d1 * phi1 - m2 * l1 * lc2 * std::pow(values[0], 2) *
+        std::sin(theta2) - phi2) / (m2 * std::pow(lc2, 2) + I2 -
+        std::pow(d2, 2) / d1);
+
+    values[2] = -(d2 * values[3] + phi1) / d1;
+
+    return values;
+  };
+
+  /**
+   * Wrap funtion is required to truncate the angle value from -180 to 180.
+   * This function will make sure that value will always be between minimum
+   * to maximum.
+   *
+   * @param value Scalar value to wrap.
+   * @param minimum Minimum range of wrap.
+   * @param maximum Maximum range of wrap.
+   */
+  double Wrap(double value,
+              const double minimum,
+              const double maximum) const
+  {
+    const double diff = maximum - minimum;
+
+    if (value > maximum)
+    {
+      value = value - diff;
+    }
+    else if (value < minimum)
+    {
+      value = value + diff;
+    }
+
+    return value;
+  };
+
+  /**
+   * This function calculates the torque for a particular action.
+   * 0 : negative torque, 1 : zero torque, 2 : positive torque.
+   *
+   * @param Action action taken.
+   */
+  double Torque(const Action& action) const
+  {
+    // Add noise to the Torque Torque is action number - 1. {0,1,2} -> {-1,0,1}.
+    return double(action - 1) + mlpack::math::Random(-0.1, 0.1);
+  }
+
+  /**
+   *
+   * This function calls the RK4 iterative method to estimate the next state
+   * based on given ordinary differential equation.
+   *
+   * @param state The current State.
+   * @param torque The torque applied.
+   */
+  arma::colvec Rk4(const arma::colvec state, const double torque) const
+  {
+    arma::colvec k1 = Dsdt(state, torque);
+    arma::colvec k2 = Dsdt(state + dt * k1 / 2, torque);
+    arma::colvec k3 = Dsdt(state + dt * k2 / 2, torque);
+    arma::colvec k4 = Dsdt(state + dt * k3, torque);
+    arma::colvec nextState = state + dt * (k1 + 2 * k2 + 2 * k3 + k4) / 6;
+
+    return nextState;
+  };
+
+ private:
+  //! Locally-stored gravity.
+  double gravity;
+
+  //! Locally-stored length of link 1.
+  double linkLength1;
+
+  //! Locally-stored length of link 2.
+  double linkLength2;
+
+  //! Locally-stored mass of link 1.
+  double linkMass1;
+
+  //! Locally-stored mass of link 2.
+  double linkMass2;
+
+  //! Locally-stored position of link 1.
+  double linkCom1;
+
+  //! Locally-stored position of link 2.
+  double linkCom2;
+
+  //! Locally-stored moment of intertia value.
+  double linkMoi;
+
+  //! Locally-stored max angular velocity of link1.
+  double maxVel1;
+
+  //! Locally-stored max angular velocity of link2.
+  double maxVel2;
+
+  //! Locally-stored dt for RK4 method.
+  double dt;
+}; // class Acrobat
+
+} // namespace rl
+} // namespace mlpack
+
+#endif

src/mlpack/tests/ann_layer_test.cpp

@@ -319,10 +319,8 @@ BOOST_AUTO_TEST_CASE(JacobianConstantLayerTest)
  */
 BOOST_AUTO_TEST_CASE(SimpleDropoutLayerTest)
 {
-  // Initialize the probability of setting a value to zero and the scale
-  // parameter.
+  // Initialize the probability of setting a value to zero.
   const double p = 0.2;
-  const double scale = 1.0 / (1.0 - p);
 
   // Initialize the input parameter.
   arma::mat input(1000, 1);
@@ -345,14 +343,8 @@ BOOST_AUTO_TEST_CASE(SimpleDropoutLayerTest)
 
   // Test the Forward function.
   module.Deterministic() = true;
-  module.Rescale() = false;
   module.Forward(std::move(input), std::move(output));
   BOOST_REQUIRE_EQUAL(arma::accu(input), arma::accu(output));
-
-  // Test the Forward function.
-  module.Rescale() = true;
-  module.Forward(std::move(input), std::move(output));
-  BOOST_REQUIRE_CLOSE(arma::accu(input) * scale, arma::accu(output), 1e-3);
 }
 
 /**
@@ -707,6 +699,70 @@ BOOST_AUTO_TEST_CASE(JacobianLeakyReLULayerTest)
   }
 }
 
+/**
+ * Jacobian FlexibleReLU module test.
+ */
+BOOST_AUTO_TEST_CASE(JacobianFlexibleReLULayerTest)
+{
+  for (size_t i = 0; i < 5; i++)
+  {
+    const size_t inputElements = math::RandInt(2, 1000);
+
+    arma::mat input;
+    input.set_size(inputElements, 1);
+
+    FlexibleReLU<> module;
+
+    double error = JacobianTest(module, input);
+    BOOST_REQUIRE_LE(error, 1e-5);
+  }
+}
+
+/**
+ * Flexible ReLU layer numerically gradient test.
+ */
+BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest)
+{
+  // Add function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction()
+    {
+      input = arma::randu(2, 1);
+      target = arma::mat("1");
+
+      model = new FFN<NegativeLogLikelihood<>, RandomInitialization>(
+          NegativeLogLikelihood<>(), RandomInitialization(0.1, 0.5));
+
+      model->Predictors() = input;
+      model->Responses() = target;
+      model->Add<LinearNoBias<> >(2, 5);
+      model->Add<FlexibleReLU<> >(0.05);
+      model->Add<LogSoftMax<> >();
+    }
+
+    ~GradientFunction()
+    {
+      delete model;
+    }
+
+    double Gradient(arma::mat& gradient) const
+    {
+      arma::mat output;
+      double error = model->Evaluate(model->Parameters(), 0, 1);
+      model->Gradient(model->Parameters(), 0, gradient, 1);
+      return error;
+    }
+
+    arma::mat& Parameters() { return model->Parameters(); }
+
+    FFN<NegativeLogLikelihood<>, RandomInitialization>* model;
+    arma::mat input, target;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
+}
+
 /**
  * Jacobian MultiplyConstant module test.
  */

src/mlpack/tests/arma_extend_test.cpp

@@ -157,7 +157,7 @@ BOOST_AUTO_TEST_CASE(MatRowColIteratorDecrementOperatorTest)
 
   // Check that postfix-- does not decrement the position when position is
   // pointing to the beginning.
-  (void) it2--;
+  auto junk = it2--; (void)(junk);
   BOOST_REQUIRE_EQUAL(it1.row(), it2.row());
   BOOST_REQUIRE_EQUAL(it1.col(), it2.col());
 

src/mlpack/tests/async_learning_test.cpp

@@ -16,7 +16,7 @@
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/reinforcement_learning/async_learning.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/cart_pole.hpp>
-#include <mlpack/core/optimizers/adam/adam_update.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/aggregated_policy.hpp>
 #include <mlpack/methods/reinforcement_learning/training_config.hpp>
@@ -35,7 +35,7 @@ BOOST_AUTO_TEST_SUITE(AsyncLearningTest);
 BOOST_AUTO_TEST_CASE(OneStepQLearningTest)
 {
   /**
-   * This is for the Travis CI server, in your own machine you shuold use more
+   * This is for the Travis CI server, in your own machine you should use more
    * threads.
    */
   #ifdef HAS_OPENMP

src/mlpack/tests/init_rules_test.cpp

@@ -24,6 +24,8 @@
 #include <mlpack/methods/ann/init_rules/const_init.hpp>
 #include <mlpack/methods/ann/init_rules/gaussian_init.hpp>
 #include <mlpack/methods/ann/init_rules/glorot_init.hpp>
+#include <mlpack/methods/ann/init_rules/he_init.hpp>
+#include <mlpack/methods/ann/init_rules/lecun_normal_init.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -328,5 +330,54 @@ BOOST_AUTO_TEST_CASE(GlorotInitNormalTest)
   BOOST_REQUIRE_EQUAL(weights3d.n_slices, 2);
 }
 
+/**
+ * Simple test of the HeInitialization class.
+ */
+BOOST_AUTO_TEST_CASE(HeInitTest)
+{
+  const size_t rows = 4;
+  const size_t cols = 4;
+  const size_t slices = 2;
+
+  arma::mat weights;
+  arma::cube weights3d;
+
+  HeInitialization initializer;
+
+  initializer.Initialize(weights, rows, cols);
+  initializer.Initialize(weights3d, rows, cols, slices);
+
+  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
+
+  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
+  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
+}
+
+/**
+ * Simple test of the LecunNormalInitialization class.
+ */
+BOOST_AUTO_TEST_CASE(LecunNormalInitTest)
+{
+  const size_t rows = 4;
+  const size_t cols = 4;
+  const size_t slices = 2;
+
+  arma::mat weights;
+  arma::cube weights3d;
+
+  LecunNormalInitialization initializer;
+
+  initializer.Initialize(weights, rows, cols);
+  initializer.Initialize(weights3d, rows, cols, slices);
+
+  BOOST_REQUIRE_EQUAL(weights.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights.n_cols, cols);
+
+  BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows);
+  BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols);
+  BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices);
+}
 
 BOOST_AUTO_TEST_SUITE_END();

src/mlpack/tests/q_learning_test.cpp

@@ -1,6 +1,7 @@
 /**
  * @file q_learning_test.hpp
  * @author Shangtong Zhang
+ * @author Rohan Raj
  *
  * Test for Q-Learning implementation
  *
@@ -17,6 +18,7 @@
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/reinforcement_learning/q_learning.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/mountain_car.hpp>
+#include <mlpack/methods/reinforcement_learning/environment/acrobat.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/cart_pole.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
 #include <mlpack/core/optimizers/adam/adam_update.hpp>
@@ -60,7 +62,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN)
   // Set up DQN agent.
   QLearning<CartPole, decltype(model), AdamUpdate, decltype(policy)>
       agent(std::move(config), std::move(model), std::move(policy),
-          std::move(replayMethod));
+      std::move(replayMethod));
 
   arma::running_stat<double> averageReturn;
   size_t episodes = 0;
@@ -133,7 +135,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN)
     // Set up the DQN agent.
     QLearning<CartPole, decltype(model), RMSPropUpdate, decltype(policy)>
         agent(std::move(config), std::move(model), std::move(policy),
-            std::move(replayMethod));
+        std::move(replayMethod));
 
     arma::running_stat<double> averageReturn;
 
@@ -170,4 +172,70 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN)
   BOOST_REQUIRE(converged);
 }
 
+//! Test DQN in Acrobat task.
+BOOST_AUTO_TEST_CASE(AcrobatWithDQN)
+{
+    // Set up the network.
+  FFN<MeanSquaredError<>, GaussianInitialization> model(MeanSquaredError<>(),
+      GaussianInitialization(0, 0.001));
+  model.Add<Linear<>>(4, 64);
+  model.Add<ReLULayer<>>();
+  model.Add<Linear<>>(64, 32);
+  model.Add<ReLULayer<>>();
+  model.Add<Linear<>>(32, 3);
+
+  // Set up the policy and replay method.
+  GreedyPolicy<Acrobat> policy(1.0, 1000, 0.1);
+  RandomReplay<Acrobat> replayMethod(20, 10000);
+
+  TrainingConfig config;
+  config.StepSize() = 0.01;
+  config.Discount() = 0.99;
+  config.TargetNetworkSyncInterval() = 100;
+  config.ExplorationSteps() = 100;
+  config.DoubleQLearning() = false;
+  config.StepLimit() = 400;
+
+  // Set up DQN agent.
+  QLearning<Acrobat, decltype(model), AdamUpdate, decltype(policy)>
+      agent(std::move(config), std::move(model), std::move(policy),
+      std::move(replayMethod));
+
+  arma::running_stat<double> averageReturn;
+  size_t episodes = 0;
+  bool converged = true;
+  while (true)
+  {
+    double episodeReturn = agent.Episode();
+    averageReturn(episodeReturn);
+    episodes += 1;
+
+    if (episodes > 1000)
+    {
+      Log::Debug << "Acrobat with DQN failed." << std::endl;
+      converged = false;
+      break;
+    }
+
+    /**
+     * I am using a thresold of -380 to check convegence.
+     */
+    Log::Debug << "Average return: " << averageReturn.mean()
+        << " Episode return: " << episodeReturn << std::endl;
+    if (averageReturn.mean() > -380.00)
+    {
+      agent.Deterministic() = true;
+      arma::running_stat<double> testReturn;
+      for (size_t i = 0; i < 20; ++i)
+        testReturn(agent.Episode());
+
+      Log::Debug << "Average return in deterministic test: "
+          << testReturn.mean() << std::endl;
+      break;
+    }
+  }
+
+  BOOST_REQUIRE(converged);
+}
+
 BOOST_AUTO_TEST_SUITE_END();

src/mlpack/tests/recurrent_network_test.cpp

@@ -12,9 +12,11 @@
 #include <mlpack/core.hpp>
 
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/rmsprop/rmsprop.hpp>
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/rnn.hpp>
 #include <mlpack/core/data/binarize.hpp>
+#include <mlpack/core/math/random.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -24,6 +26,7 @@
 using namespace mlpack;
 using namespace mlpack::ann;
 using namespace mlpack::optimization;
+using namespace mlpack::math;
 
 BOOST_AUTO_TEST_SUITE(RecurrentNetworkTest);
 
@@ -1085,4 +1088,139 @@ BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest)
   ReberGrammarTestCustomNetwork(16, true);
 }
 
+/**
+ * @brief Generates noisy sine wave and outputs the data and the labels that
+ *        can be used directly for training and testing with RNN.
+ *
+ * @param data The data points as output
+ * @param labels The expected values as output
+ * @param rho The size of the sequence of each data point
+ * @param outputSteps How many output steps to consider for every rho inputs
+ * @param dataPoints  The number of generated data points. The actual generated
+ *        data points may be more than this to adjust to the outputSteps. But at
+ *        the minimum these many data points will be generated.
+ * @param gain The gain on the amplitude
+ * @param freq The frquency of the sine wave
+ * @param phase The phase shift if any
+ * @param noisePercent The percent noise to induce
+ * @param numCycles How many full size wave cycles required. All the data
+ *        points will be fit into these cycles.
+ * @param normalize Whether to normalise the data. This may be required for some
+ *        layers like LSTM. Default is true.
+ */
+void GenerateNoisySinRNN(arma::cube& data,
+                         arma::cube& labels,
+                         size_t rho,
+                         size_t outputSteps = 1,
+                         const int dataPoints = 100,
+                         const double gain = 1.0,
+                         const int freq = 10,
+                         const double phase = 0,
+                         const int noisePercent = 20,
+                         const double numCycles = 6.0,
+                         const bool normalize = true)
+{
+  int points = dataPoints;
+  int r = dataPoints % rho;
+
+  if (r == 0)
+  {
+    points += outputSteps;
+  }
+  else
+  {
+    points += rho - r + outputSteps;
+  }
+
+  arma::colvec x(points);
+  int i = 0;
+  double interval = numCycles / freq / points;
+
+  x.for_each([&i, gain, freq, phase, noisePercent, interval]
+    (arma::colvec::elem_type& val) {
+    double t = interval * (i++);
+    val = gain * ::sin(2 * M_PI * freq * t + phase) +
+        (noisePercent * gain / 100 * Random(0.0, 0.1));
+  });
+
+  arma::colvec y = x;
+  if (normalize)
+    y = arma::normalise(x);
+
+  // Now break this into columns of rho size slices.
+  size_t numColumns = y.n_elem / rho;
+  data = arma::cube(1, numColumns, rho);
+  labels = arma::cube(outputSteps, numColumns, 1);
+
+  for (size_t i = 0; i < numColumns; ++i)
+  {
+    data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1);
+    labels.subcube(0, i, 0, outputSteps - 1, i, 0) =
+        y.rows(i * rho + rho, i * rho + rho + outputSteps - 1);
+  }
+}
+
+/**
+ * @brief RNNSineTest Test a simple RNN using noisy sine. Use single output
+ *        for multiple inputs.
+ * @param hiddenUnits No of units in the hiddenlayer.
+ * @param rho The input sequence length.
+ * @param numEpochs The number of epochs to run.
+ * @return The mean squared error of the prediction.
+ */
+double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100)
+{
+  RNN<MeanSquaredError<> > net(rho, true);
+  net.Add<LinearNoBias<> >(1, hiddenUnits);
+  net.Add<LSTM<> >(hiddenUnits, hiddenUnits);
+  net.Add<LinearNoBias<> >(hiddenUnits, 1);
+
+  RMSProp opt(0.005, 100, 0.9, 1e-08, 50000, 1e-5);
+
+  // Generate data
+  arma::cube data;
+  arma::cube labels;
+  GenerateNoisySinRNN(data, labels, rho, 1, 2000, 20.0, 200, 0.0, 45, 20);
+
+  // Break into training and test sets. Simply split along columns.
+  size_t trainCols = data.n_cols * 0.8; // Take 20% out for testing.
+  size_t testCols = data.n_cols - trainCols;
+  arma::cube testData = data.subcube(0, data.n_cols - testCols, 0,
+      data.n_rows - 1, data.n_cols - 1, data.n_slices - 1);
+  arma::cube testLabels = labels.subcube(0, labels.n_cols - testCols, 0,
+      labels.n_rows - 1, labels.n_cols - 1, labels.n_slices - 1);
+
+  for (size_t i = 0; i < numEpochs; ++i)
+  {
+    net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1,
+        data.n_slices - 1), labels.subcube(0, 0, 0, labels.n_rows - 1,
+        trainCols - 1, labels.n_slices - 1), opt);
+  }
+  // Well now it should be trained. Do the test here.
+  arma::cube prediction;
+  net.Predict(testData, prediction);
+
+  // The prediction must really follow the test data. So convert both the test
+  // data and the pediction to vectors and compare the two.
+  arma::colvec testVector = arma::vectorise(testData);
+  arma::colvec predVector = arma::vectorise(prediction);
+
+  // Adjust the vectors for comparison, as the prediction is one step ahead.
+  testVector = testVector.rows(1, testVector.n_rows - 1);
+  predVector = predVector.rows(0, predVector.n_rows - 2);
+  double error = std::sqrt(arma::sum(arma::square(testVector - predVector))) /
+      testVector.n_rows;
+
+  return error;
+}
+
+/**
+ * Test RNN using multiple timestep input and single output.
+ */
+BOOST_AUTO_TEST_CASE(MultiTimestepTest)
+{
+  double err = RNNSineTest(4, 10, 20);
+  BOOST_REQUIRE_LE(err, 1e-02);
+}
+
 BOOST_AUTO_TEST_SUITE_END();

src/mlpack/tests/rl_components_test.cpp

@@ -14,6 +14,7 @@
 
 #include <mlpack/methods/reinforcement_learning/environment/mountain_car.hpp>
 #include <mlpack/methods/reinforcement_learning/environment/cart_pole.hpp>
+#include <mlpack/methods/reinforcement_learning/environment/acrobat.hpp>
 #include <mlpack/methods/reinforcement_learning/replay/random_replay.hpp>
 #include <mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp>
 
@@ -25,6 +26,23 @@ using namespace mlpack::rl;
 
 BOOST_AUTO_TEST_SUITE(RLComponentsTest)
 
+/**
+ * Constructs a Acrobat instance and check if the main rountine works as
+ * it should be.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAcrobatTest)
+{
+  const Acrobat task = Acrobat();
+
+  Acrobat::State state = task.InitialSample();
+  Acrobat::Action action = Acrobat::Action::negativeTorque;
+  double reward = task.Sample(state, action);
+
+  BOOST_REQUIRE_EQUAL(reward, -1.0);
+  BOOST_REQUIRE(!task.IsTerminal(state));
+  BOOST_REQUIRE_EQUAL(3, Acrobat::Action::size);
+}
+
 /**
  * Constructs a MountainCar instance and check if the main rountine works as
  * it should be.