diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index 9ae00acee7..04171dc955 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -242,6 +242,7 @@ * - Moksh Jain * - Manthan-R-Sheth * - Namrata Mukhija + * - Rohan Raj */ // First, include all of the prerequisites. diff --git a/src/mlpack/methods/ann/init_rules/CMakeLists.txt b/src/mlpack/methods/ann/init_rules/CMakeLists.txt index f43194e92e..7d18f987fe 100644 --- a/src/mlpack/methods/ann/init_rules/CMakeLists.txt +++ b/src/mlpack/methods/ann/init_rules/CMakeLists.txt @@ -3,8 +3,10 @@ set(SOURCES const_init.hpp gaussian_init.hpp + he_init.hpp init_rules_traits.hpp kathirvalavakumar_subavathi_init.hpp + lecun_normal_init.hpp network_init.hpp nguyen_widrow_init.hpp oivs_init.hpp diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp new file mode 100644 index 0000000000..421b499dc3 --- /dev/null +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -0,0 +1,106 @@ +/** + * @file he_init.hpp + * @author Dakshit Agrawal + * @author Prabhat Sharma + * + * Intialization rule given by He et. al. for neural networks. The He + * initialization initializes weights of the neural network to better + * suit the rectified activation units. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP +#define MLPACK_METHODS_ANN_INIT_RULES_HE_INIT_HPP + +#include +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * This class is used to initialize weight matrix with the He + * initialization rule given by He et. al. for neural networks. The He + * initialization initializes weights of the neural network to better + * suit the rectified activation units. + * + * For more information, the following paper can be referred to: + * + * @code + * @article{Delving2015, + * title = {Delving Deep into Rectifiers: Surpassing Human-Level Performance + * on ImageNet Classification}, + * author = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun}, + * journal = {2015 IEEE International Conference on Computer Vision (ICCV)}, + * year = {2015}, + * pages = {1026-1034} + * } + * @endcode + * + */ +class HeInitialization +{ + public: + /** + * Initialize the HeInitialization object. + */ + HeInitialization() + { + // Nothing to do here. + } + + /** + * Initialize the elements of the weight matrix with the He initialization + * rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + */ + void Initialize(arma::mat& W, const size_t rows, const size_t cols) + { + // He initialization rule says to initialize weights with random + // values taken from a gaussian distribution with mean = 0 and + // standard deviation = sqrt(2/rows), i.e. variance = (2/rows). + const double variance = 2.0 / (double)rows; + + if (W.is_empty()) + { + W.set_size(rows, cols); + } + + // Multipling a random variable X with variance V(X) by some factor c, + // then the variance V(cX) = (c^2) * V(X). + W.imbue( [&]() { return sqrt(variance) * arma::randn(); } ); + } + + /** + * Initialize the elements of the specified weight 3rd order tensor + * with He initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + * @param slice Numbers of slices. + */ + void Initialize(arma::cube & W, + const size_t rows, + const size_t cols, + const size_t slices) + { + if (W.is_empty()) + W.set_size(rows, cols, slices); + + for (size_t i = 0; i < slices; i++) + Initialize(W.slice(i), rows, cols); + } +}; // class HeInitialization + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp new file mode 100644 index 0000000000..19bfb46173 --- /dev/null +++ b/src/mlpack/methods/ann/init_rules/lecun_normal_init.hpp @@ -0,0 +1,112 @@ +/** + * @file lecun_normal_init.hpp + * @author Dakshit Agrawal + * @author Prabhat Sharma + * + * Intialization rule given by Lecun et. al. for neural networks and + * also mentioned in Self Normalizing Networks. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP +#define MLPACK_METHODS_ANN_INIT_RULES_LECUN_NORMAL_INIT_HPP + +#include +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * This class is used to initialize weight matrix with the Lecun Normalization + * initialization rule. + * + * For more information, the following papers can be referred to: + * + * @code + * @inproceedings{Klambauer2017, + * itle = {Self-Normalizing Neural Networks.}, + * author = {Klambauer, Günter and Unterthiner, Thomas + * and Mayr, Andreas and Hochreiter, Sepp}, + * pages = {972-981}, + * year = {2017} + * } + * + * @inproceedings{LeCun1998, + * title = {Efficient BackProp}, + * author = {LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B. + * and M\"{u}ller, Klaus-Robert}, + * year = {1998}, + * pages = {9--50} + * } + * @endcode + * +*/ +class LecunNormalInitialization +{ + public: + /** + * Initialize the LecunNormalInitialization object. + */ + LecunNormalInitialization() + { + // Nothing to do here. + } + + /** + * Initialize the elements of the weight matrix with the Lecun + * Normal initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + */ + void Initialize(arma::mat& W, + const size_t rows, + const size_t cols) + { + // He initialization rule says to initialize weights with random + // values taken from a gaussian distribution with mean = 0 and + // standard deviation = sqrt(1 / rows), i.e. variance = (1 / rows). + const double variance = 1.0 / ((double) rows); + + if (W.is_empty()) + { + W.set_size(rows, cols); + } + + // Multipling a random variable X with variance V(X) by some factor c, + // then the variance V(cX) = (c ^ 2) * V(X). + W.imbue( [&]() { return sqrt(variance) * arma::randn(); } ); + } + + /** + * Initialize the elements of the specified weight 3rd order tensor + * with Lecun Normal initialization rule. + * + * @param W Weight matrix to initialize. + * @param rows Number of rows. + * @param cols Number of columns. + * @param slice Numbers of slices. + */ + void Initialize(arma::cube & W, + const size_t rows, + const size_t cols, + const size_t slices) + { + if (W.is_empty()) + W.set_size(rows, cols, slices); + + for (size_t i = 0; i < slices; i++) + Initialize(W.slice(i), rows, cols); + } +}; // class LecunNormalInitialization + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index de2a339a8d..1a8129115f 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -30,6 +30,8 @@ set(SOURCES elu_impl.hpp fast_lstm.hpp fast_lstm_impl.hpp + flexible_relu.hpp + flexible_relu_impl.hpp glimpse.hpp glimpse_impl.hpp gru.hpp diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp index d562b85cf1..a388820c71 100644 --- a/src/mlpack/methods/ann/layer/dropout.hpp +++ b/src/mlpack/methods/ann/layer/dropout.hpp @@ -3,7 +3,7 @@ * @author Marcus Edel * * Definition of the Dropout class, which implements a regularizer that - * randomly sets units to zero. Preventing units from co-adapting. + * randomly sets units to zero preventing units from co-adapting. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -18,12 +18,12 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { + /** - * The dropout layer is a regularizer that randomly with probability ratio + * The dropout layer is a regularizer that randomly with probability 'ratio' * sets input values to zero and scales the remaining elements by factor 1 / - * (1 - ratio). If rescale is true the input is scaled with 1 / (1-p) when - * deterministic is false. In the deterministic mode (during testing), the layer - * just scales the output. + * (1 - ratio) rather than during test time so as to keep the expected sum same. + * In the deterministic mode (during testing), there is no change in the input. * * Note: During training you should set deterministic to false and during * testing you should set deterministic to true. @@ -47,21 +47,17 @@ namespace ann /** Artificial Neural Network. */ { * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ -template < - typename InputDataType = arma::mat, - typename OutputDataType = arma::mat -> +template class Dropout { public: /** - * Create the Dropout object using the specified ratio and rescale - * parameter. + * Create the Dropout object using the specified ratio parameter. * * @param ratio The probability of setting a value to zero. - * @param rescale If true the input is rescaled when deterministic is False. */ - Dropout(const double ratio = 0.5, const bool rescale = true); + Dropout(const double ratio = 0.5); /** * Ordinary feed forward pass of the dropout layer. @@ -85,7 +81,7 @@ class Dropout arma::Mat&& g); //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } + InputDataType const& InputParameter() const { return inputParameter; } //! Modify the input parameter. InputDataType& InputParameter() { return inputParameter; } @@ -114,11 +110,6 @@ class Dropout scale = 1.0 / (1.0 - ratio); } - //! The value of the rescale parameter. - bool Rescale() const {return rescale; } - //! Modify the value of the rescale parameter. - bool& Rescale() {return rescale; } - /** * Serialize the layer. */ @@ -146,9 +137,6 @@ class Dropout //! If true dropout and scaling is disabled, see notes above. bool deterministic; - - //! If true the input is rescaled when deterministic is False. - bool rescale; }; // class Dropout } // namespace ann diff --git a/src/mlpack/methods/ann/layer/dropout_impl.hpp b/src/mlpack/methods/ann/layer/dropout_impl.hpp index 2f7c03c6e9..ccac9588f8 100644 --- a/src/mlpack/methods/ann/layer/dropout_impl.hpp +++ b/src/mlpack/methods/ann/layer/dropout_impl.hpp @@ -21,11 +21,10 @@ namespace ann /** Artificial Neural Network. */ { template Dropout::Dropout( - const double ratio, const bool rescale) : + const double ratio) : ratio(ratio), scale(1.0 / (1.0 - ratio)), - deterministic(true), - rescale(rescale) + deterministic(false) { // Nothing to do here. } @@ -40,21 +39,14 @@ void Dropout::Forward( // (during testing). if (deterministic) { - if (!rescale) - { - output = input; - } - else - { - output = input * scale; - } + output = input; } else { // Scale with input / (1 - ratio) and set values to zero with probability - // ratio. + // 'ratio'. mask = arma::randu >(input.n_rows, input.n_cols); - mask.transform( [&](double val) { return (val > ratio); } ); + mask.transform([&](double val) { return (val > ratio); }); output = input % mask * scale; } } @@ -76,7 +68,6 @@ void Dropout::serialize( const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(ratio); - ar & BOOST_SERIALIZATION_NVP(rescale); // Reset scale. scale = 1.0 / (1.0 - ratio); diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp new file mode 100644 index 0000000000..a9c837e022 --- /dev/null +++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp @@ -0,0 +1,175 @@ +/** + * @file flexible_relu.hpp + * @author Aarush Gupta + * @author Manthan-R-Sheth + * + * Definition of FlexibleReLU layer as described by + * Suo Qiu, Xiangmin Xu and Bolun Cai in + * "FReLU: Flexible Rectified Linear Units for Improving Convolutional + * Neural Networks", 2018 + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_HPP +#define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_HPP + +#include + +namespace mlpack { +namespace ann /**Artificial Neural Network*/ { + +/** + * The FlexibleReLU activation function, defined by + * + * @f{eqnarray*}{ + * f(x) &=& \max(0,x)+alpha \\ + * f'(x) &=& \left\{ + * \begin(array){lr} + * 1 & : x > 0 \\ + * 0 & : x \le 0 + * \end{array} + * \right + * @f} + * + * For more information, read the following paper: + * + * @code + * @article{Qiu2018, + * author = {Suo Qiu, Xiangmin Xu and Bolun Cai}, + * title = {FReLU: Flexible Rectified Linear Units for Improving + * Convolutional Neural Networks} + * journal = {arxiv preprint}, + * URL = {https://arxiv.org/abs/1706.08098}, + * year = {2018} + * } + * @endcode + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mar, + * arma::sp_mat or arma::cube) + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube) + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class FlexibleReLU +{ + public: + /** + * + * Create the FlexibleReLU object using the specified parameters. + * The non zero parameter can be adjusted by specifying the parameter + * alpha which controls the range of the relu function. (Default alpha = 0) + * This parameter is trainable. + * + * @param alpha Parameter for adjusting the range of the relu function. + * + */ + FlexibleReLU(const double alpha = 0); + + /** + * Reset the layer parameter. + */ + void Reset(); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const InputType&& input, OutputType&& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards through f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const DataType&& input, DataType&& gy, DataType&& g); + + /** + * Calculate the gradient using the output delta and the input activation. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + */ + template + void Gradient(const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); + + //! Get the parameters. + OutputDataType const& Parameters() const { return alpha; } + //! Modify the parameters. + OutputDataType& Parameters() { return alpha; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta;} + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the parameter controlling the range of the relu function. + double const& Alpha() const { return alpha; } + //! Modify the parameter controlling the range of the relu function. + double& Alpha() { return alpha; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version*/); + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Parameter object. + OutputDataType alpha; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Parameter controlling the range of the rectifier function + double userAlpha; +}; // class FlexibleReLU + +} // namespace ann +} // namespace mlpack + +// Include implementation +#include "flexible_relu_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp new file mode 100644 index 0000000000..52ab56f498 --- /dev/null +++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp @@ -0,0 +1,86 @@ +/** + * @file flexible_relu_impl.hpp + * @author Aarush Gupta + * @author Manthan-R-Sheth + * + * Implementation of FlexibleReLU layer as described by + * Suo Qiu, Xiangmin Xu and Bolun Cai in + * "FReLU: Flexible Rectified Linear Units for Improving Convolutional + * Neural Networks", 2018 + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP + +#include "flexible_relu.hpp" +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +FlexibleReLU::FlexibleReLU( + const double alpha) : userAlpha(alpha) +{ + this->alpha.set_size(1, 1); + this->alpha(0) = userAlpha; +} + +template +void FlexibleReLU::Reset() +{ + //! Set value of alpha to the one given by user. + alpha(0) = userAlpha; +} + +template +template +void FlexibleReLU::Forward( + const InputType&& input, OutputType&& output) +{ + output = arma::clamp(input, 0.0, DBL_MAX) + alpha(0); +} + +template +template +void FlexibleReLU::Backward( + const DataType&& input, DataType&& gy, DataType&& g) +{ + //! Compute the first derivative of FlexibleReLU function. + g = gy % arma::clamp(arma::sign(input), 0.0, 1.0); +} + +template +template +void FlexibleReLU::Gradient( + const arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient) +{ + if (gradient.n_elem == 0) + { + gradient.set_size(1, 1); + } + + gradient(0) = arma::accu(error) / input.n_cols; +} + + +template +template +void FlexibleReLU::serialize( + Archive& ar, + const unsigned int /* version*/) +{ + ar & BOOST_SERIALIZATION_NVP(alpha); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 96c1280079..4b68538836 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -126,6 +127,7 @@ using LayerTypes = boost::variant< Dropout*, AlphaDropout*, ELU*, + FlexibleReLU*, Glimpse*, HardTanH*, Join*, diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index 88d2b662d7..dfa7af8e92 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -103,7 +103,7 @@ class LeakyReLU private: /** - * Computes the LeakReLU function + * Computes the LeakyReLU function * * @param x Input data. * @return f(x). @@ -114,7 +114,7 @@ class LeakyReLU } /** - * Computes the Leaky ReLU function using a dense matrix as input. + * Computes the LeakyReLU function using a dense matrix as input. * * @param x Input data. * @param y The resulting output activation. diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp index 84c37b0a97..8e86ef035c 100644 --- a/src/mlpack/methods/ann/rnn_impl.hpp +++ b/src/mlpack/methods/ann/rnn_impl.hpp @@ -201,6 +201,7 @@ double RNN::Evaluate( ResetCells(); double performance = 0; + size_t responseSeq = 0; for (size_t seqNum = 0; seqNum < rho; ++seqNum) { @@ -208,8 +209,10 @@ double RNN::Evaluate( arma::mat stepData(predictors.slice(seqNum).colptr(begin), predictors.n_rows, batchSize, false, true); Forward(std::move(stepData)); - arma::mat respData(responses.slice(seqNum).colptr(begin), - responses.n_rows, batchSize, false, true); + if (!single) + { + responseSeq = seqNum; + } if (!deterministic) { @@ -222,7 +225,7 @@ double RNN::Evaluate( performance += outputLayer.Forward(std::move(boost::apply_visitor( outputParameterVisitor, network.back())), - std::move(arma::mat(responses.slice(seqNum).colptr(begin), + std::move(arma::mat(responses.slice(responseSeq).colptr(begin), responses.n_rows, batchSize, false, true))); } @@ -283,19 +286,25 @@ void RNN::Gradient( { error.zeros(); } + else if (single && seqNum == 0) + { + outputLayer.Backward(std::move(boost::apply_visitor( + outputParameterVisitor, network.back())), + std::move(arma::mat(responses.slice(0).colptr(begin), + responses.n_rows, batchSize, false, true)), std::move(error)); + } else { outputLayer.Backward(std::move(boost::apply_visitor( outputParameterVisitor, network.back())), std::move(arma::mat(responses.slice(rho - seqNum - 1).colptr(begin), - responses.n_rows, batchSize, false, true)), - std::move(error)); + responses.n_rows, batchSize, false, true)), std::move(error)); } Backward(); Gradient(std::move( arma::mat(predictors.slice(rho - seqNum - 1).colptr(begin), - predictors.n_rows, batchSize, false, true))); + predictors.n_rows, batchSize, false, true))); gradient += currentGradient; } } diff --git a/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt index 58baa7e14d..6814fbab5d 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt +++ b/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt @@ -3,6 +3,7 @@ set(SOURCES mountain_car.hpp cart_pole.hpp + acrobat.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp new file mode 100644 index 0000000000..713121ae80 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobat.hpp @@ -0,0 +1,345 @@ +/** + * @file acrobat.hpp + * @author Rohan Raj + * + * This file is an implementation of Acrobat task: + * https://gym.openai.com/envs/Acrobot-v1/ + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_ENVIRONMENT_ACROBAT_HPP +#define MLPACK_METHODS_RL_ENVIRONMENT_ACROBAT_HPP + +#include + +namespace mlpack{ +namespace rl{ + +/** + * Implementation of Acrobat game. Acrobot is a 2-link pendulum with only the + * second joint actuated. Intitially, both links point downwards. The goal is + * to swing the end-effector at a height at least the length of one link above + * the base. Both links can swing freely and can pass by each other, i.e., + * they don't collide when they have the same angle. + */ +class Acrobat +{ + public: + /* + * Implementation of Acrobat State. Each State is a tuple vector + * (theta1, thetha2, angular velocity 1, angular velocity 2). + */ + class State + { + public: + /** + * Construct a state instance. + */ + State(): data(dimension) { /* nothing to do here */ } + + /** + * Construct a state instance from given data. + * + * @param data Data for the theta and angular velocity of two links. + */ + State(const arma::colvec& data) : data(data) + { /* nothing to do here */ } + + //! Modify the state representation. + arma::colvec& Data() { return data; } + + //! Get value of theta (one). + double Theta1() const { return data[0]; } + //! Modify value of theta (one). + double& Theta1() { return data[0]; } + + //! Get value of theta (two). + double Theta2() const { return data[1]; } + //! Modify value of theta (two). + double& Theta2() { return data[1]; } + + //! Get value of Angular velocity (one). + double AngularVelocity1() const { return data[2]; } + //! Modify the angular velocity (one). + double& AngularVelocity1() { return data[2]; } + + //! Get value of Angular velocity (two). + double AngularVelocity2() const { return data[3]; } + //! Modify the angular velocity (two). + double& AngularVelocity2() { return data[3]; } + + //! Encode the state to a column vector. + const arma::colvec& Encode() const { return data; } + + //! Dimension of the encoded state. + static constexpr size_t dimension = 4; + + private: + //! Locally-Stored (theta1, theta2, angular velocity 1, angular velocity2). + arma::colvec data; + }; + + /* + * Implementation of action for Acrobat + */ + enum Action + { + negativeTorque, + zeroTorque, + positiveTorque, + + // Track the size of the action space. + size + }; + + /** + * Construct a Acrobat instance using the given constants. + * + * @param gravity The gravity parameter. + * @param linkLength1 The length of link 1. + * @param linkLength2 The length of link 2. + * @param linkMass1 The mass of link 1. + * @param linkMass2 The mass of link 2. + * @param linkCom1 The position of the center of mass of link 1. + * @param linkCom2 The position of the center of mass of link 2. + * @param linkMoi The moments of inertia for both link. + * @param maxVel1 The max angular velocity of link1. + * @param maxVel2 The max angular velocity of link2. + * @param dt The differential value. + */ + Acrobat(const double gravity = 9.81, + const double linkLength1 = 1.0, + const double linkLength2 = 1.0, + const double linkMass1 = 1.0, + const double linkMass2 = 1.0, + const double linkCom1 = 0.5, + const double linkCom2 = 0.5, + const double linkMoi = 1.0, + const double maxVel1 = 4 * M_PI, + const double maxVel2 = 9 * M_PI, + const double dt = 0.2) : + gravity(gravity), + linkLength1(linkLength1), + linkLength2(linkLength2), + linkMass1(linkMass1), + linkMass2(linkMass2), + linkCom1(linkCom1), + linkCom2(linkCom2), + linkMoi(linkMoi), + maxVel1(maxVel1), + maxVel2(maxVel2), + dt(dt) + { /* Nothing to do here */ } + + /** + * Dynamics of the Acrobat System. To get reward and next state based on + * current state and current action. Always return -1 reward. + * + * @param state The current State. + * @param action The action taken. + * @param nextState The next state. + * @return reward, it's always -1.0. + */ + double Sample(const State& state, + const Action& action, + State& nextState) const + { + // Make a vector to estimate nextstate. + arma::colvec currentState = {state.Theta1(), state.Theta2(), + state.AngularVelocity1(), state.AngularVelocity2()}; + + arma::colvec currentNextState = Rk4(currentState, Torque(action)); + + nextState.Theta1() = Wrap(currentNextState[0], -M_PI, M_PI); + + nextState.Theta2() = Wrap(currentNextState[1], -M_PI, M_PI); + //! The value of angular velocity is bounded in min and max value. + nextState.AngularVelocity1() = std::min( + std::max(currentNextState[2], -maxVel1), maxVel1); + nextState.AngularVelocity2() = std::min( + std::max(currentNextState[3], -maxVel2), maxVel2); + + return -1.0; + }; + + /** + * Dynamics of the Acrobat System. To get reward and next state based on + * current state and current action. This function calls the Sample function + * to estimate the next state return reward for taking a particular action. + * + * @param state The current State. + * @param action The action taken. + * @param nextState The next state. + */ + double Sample(const State& state, const Action& action) const + { + State nextState; + return Sample(state, action, nextState); + } + + /** + * This function does random initialization of state space. + */ + State InitialSample() const + { + return State((arma::randu(4) - 0.5) / 5.0); + } + + /** + * This function checks if the acrobat has reached the terminal state. + * + * @param state The current State. + */ + bool IsTerminal(const State& state) const + { + return bool (-std::cos(state.Theta1())-std::cos(state.Theta1() + + state.Theta2()) > 1.0); + } + + /** + * This is the ordinary differential equations required for estimation of + * nextState through RK4 method. + * + * @param state Current State. + * @param torque The torque Applied. + */ + arma::colvec Dsdt(arma::colvec state, const double torque) const + { + const double m1 = linkMass1; + const double m2 = linkMass2; + const double l1 = linkLength1; + const double lc1 = linkCom1; + const double lc2 = linkCom2; + const double I1 = linkMoi; + const double I2 = linkMoi; + const double g = gravity; + const double a = torque; + const double theta1 = state[0]; + const double theta2 = state[1]; + + arma::colvec values(4); + values[0] = state[2]; + values[1] = state[3]; + + const double d1 = m1 * std::pow(lc1, 2) + m2 * (std::pow(l1, 2) + + std::pow(lc2, 2) + 2 * l1 * lc2 * std::cos(theta2)) + I1 + I2; + + const double d2 = m2 * (std::pow(lc2, 2) + l1 * lc2 * std::cos(theta2)) + + I2; + + const double phi2 = m2 * lc2 * g * std::cos(theta1 + theta2 - M_PI / 2.); + + const double phi1 = - m2 * l1 * lc2 * std::pow(values[1], 2) * + std::sin(theta2) - 2 * m2 * l1 * lc2 * values[1] * values[0] * + std::sin(theta2) + (m1 * lc1 + m2 * l1) * g * + std::cos(theta1 - M_PI / 2) + phi2; + + values[3] = (a + d2 / d1 * phi1 - m2 * l1 * lc2 * std::pow(values[0], 2) * + std::sin(theta2) - phi2) / (m2 * std::pow(lc2, 2) + I2 - + std::pow(d2, 2) / d1); + + values[2] = -(d2 * values[3] + phi1) / d1; + + return values; + }; + + /** + * Wrap funtion is required to truncate the angle value from -180 to 180. + * This function will make sure that value will always be between minimum + * to maximum. + * + * @param value Scalar value to wrap. + * @param minimum Minimum range of wrap. + * @param maximum Maximum range of wrap. + */ + double Wrap(double value, + const double minimum, + const double maximum) const + { + const double diff = maximum - minimum; + + if (value > maximum) + { + value = value - diff; + } + else if (value < minimum) + { + value = value + diff; + } + + return value; + }; + + /** + * This function calculates the torque for a particular action. + * 0 : negative torque, 1 : zero torque, 2 : positive torque. + * + * @param Action action taken. + */ + double Torque(const Action& action) const + { + // Add noise to the Torque Torque is action number - 1. {0,1,2} -> {-1,0,1}. + return double(action - 1) + mlpack::math::Random(-0.1, 0.1); + } + + /** + * + * This function calls the RK4 iterative method to estimate the next state + * based on given ordinary differential equation. + * + * @param state The current State. + * @param torque The torque applied. + */ + arma::colvec Rk4(const arma::colvec state, const double torque) const + { + arma::colvec k1 = Dsdt(state, torque); + arma::colvec k2 = Dsdt(state + dt * k1 / 2, torque); + arma::colvec k3 = Dsdt(state + dt * k2 / 2, torque); + arma::colvec k4 = Dsdt(state + dt * k3, torque); + arma::colvec nextState = state + dt * (k1 + 2 * k2 + 2 * k3 + k4) / 6; + + return nextState; + }; + + private: + //! Locally-stored gravity. + double gravity; + + //! Locally-stored length of link 1. + double linkLength1; + + //! Locally-stored length of link 2. + double linkLength2; + + //! Locally-stored mass of link 1. + double linkMass1; + + //! Locally-stored mass of link 2. + double linkMass2; + + //! Locally-stored position of link 1. + double linkCom1; + + //! Locally-stored position of link 2. + double linkCom2; + + //! Locally-stored moment of intertia value. + double linkMoi; + + //! Locally-stored max angular velocity of link1. + double maxVel1; + + //! Locally-stored max angular velocity of link2. + double maxVel2; + + //! Locally-stored dt for RK4 method. + double dt; +}; // class Acrobat + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 521efc6169..d6d1478a82 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -319,10 +319,8 @@ BOOST_AUTO_TEST_CASE(JacobianConstantLayerTest) */ BOOST_AUTO_TEST_CASE(SimpleDropoutLayerTest) { - // Initialize the probability of setting a value to zero and the scale - // parameter. + // Initialize the probability of setting a value to zero. const double p = 0.2; - const double scale = 1.0 / (1.0 - p); // Initialize the input parameter. arma::mat input(1000, 1); @@ -345,14 +343,8 @@ BOOST_AUTO_TEST_CASE(SimpleDropoutLayerTest) // Test the Forward function. module.Deterministic() = true; - module.Rescale() = false; module.Forward(std::move(input), std::move(output)); BOOST_REQUIRE_EQUAL(arma::accu(input), arma::accu(output)); - - // Test the Forward function. - module.Rescale() = true; - module.Forward(std::move(input), std::move(output)); - BOOST_REQUIRE_CLOSE(arma::accu(input) * scale, arma::accu(output), 1e-3); } /** @@ -707,6 +699,70 @@ BOOST_AUTO_TEST_CASE(JacobianLeakyReLULayerTest) } } +/** + * Jacobian FlexibleReLU module test. + */ +BOOST_AUTO_TEST_CASE(JacobianFlexibleReLULayerTest) +{ + for (size_t i = 0; i < 5; i++) + { + const size_t inputElements = math::RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElements, 1); + + FlexibleReLU<> module; + + double error = JacobianTest(module, input); + BOOST_REQUIRE_LE(error, 1e-5); + } +} + +/** + * Flexible ReLU layer numerically gradient test. + */ +BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest) +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input = arma::randu(2, 1); + target = arma::mat("1"); + + model = new FFN, RandomInitialization>( + NegativeLogLikelihood<>(), RandomInitialization(0.1, 0.5)); + + model->Predictors() = input; + model->Responses() = target; + model->Add >(2, 5); + model->Add >(0.05); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + arma::mat output; + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, RandomInitialization>* model; + arma::mat input, target; + } function; + + BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); +} + /** * Jacobian MultiplyConstant module test. */ diff --git a/src/mlpack/tests/arma_extend_test.cpp b/src/mlpack/tests/arma_extend_test.cpp index 175528ea7f..0594fb567b 100644 --- a/src/mlpack/tests/arma_extend_test.cpp +++ b/src/mlpack/tests/arma_extend_test.cpp @@ -157,7 +157,7 @@ BOOST_AUTO_TEST_CASE(MatRowColIteratorDecrementOperatorTest) // Check that postfix-- does not decrement the position when position is // pointing to the beginning. - (void) it2--; + auto junk = it2--; (void)(junk); BOOST_REQUIRE_EQUAL(it1.row(), it2.row()); BOOST_REQUIRE_EQUAL(it1.col(), it2.col()); diff --git a/src/mlpack/tests/async_learning_test.cpp b/src/mlpack/tests/async_learning_test.cpp index feb16e4ce2..c937771164 100644 --- a/src/mlpack/tests/async_learning_test.cpp +++ b/src/mlpack/tests/async_learning_test.cpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_SUITE(AsyncLearningTest); BOOST_AUTO_TEST_CASE(OneStepQLearningTest) { /** - * This is for the Travis CI server, in your own machine you shuold use more + * This is for the Travis CI server, in your own machine you should use more * threads. */ #ifdef HAS_OPENMP diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp index d1a9087478..692bf57302 100644 --- a/src/mlpack/tests/init_rules_test.cpp +++ b/src/mlpack/tests/init_rules_test.cpp @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include #include "test_tools.hpp" @@ -328,5 +330,54 @@ BOOST_AUTO_TEST_CASE(GlorotInitNormalTest) BOOST_REQUIRE_EQUAL(weights3d.n_slices, 2); } +/** + * Simple test of the HeInitialization class. + */ +BOOST_AUTO_TEST_CASE(HeInitTest) +{ + const size_t rows = 4; + const size_t cols = 4; + const size_t slices = 2; + + arma::mat weights; + arma::cube weights3d; + + HeInitialization initializer; + + initializer.Initialize(weights, rows, cols); + initializer.Initialize(weights3d, rows, cols, slices); + + BOOST_REQUIRE_EQUAL(weights.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights.n_cols, cols); + + BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols); + BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); +} + +/** + * Simple test of the LecunNormalInitialization class. + */ +BOOST_AUTO_TEST_CASE(LecunNormalInitTest) +{ + const size_t rows = 4; + const size_t cols = 4; + const size_t slices = 2; + + arma::mat weights; + arma::cube weights3d; + + LecunNormalInitialization initializer; + + initializer.Initialize(weights, rows, cols); + initializer.Initialize(weights3d, rows, cols, slices); + + BOOST_REQUIRE_EQUAL(weights.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights.n_cols, cols); + + BOOST_REQUIRE_EQUAL(weights3d.n_rows, rows); + BOOST_REQUIRE_EQUAL(weights3d.n_cols, cols); + BOOST_REQUIRE_EQUAL(weights3d.n_slices, slices); +} BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 600288c966..6e7613d70b 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -1,6 +1,7 @@ /** * @file q_learning_test.hpp * @author Shangtong Zhang + * @author Rohan Raj * * Test for Q-Learning implementation * @@ -17,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -60,7 +62,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN) // Set up DQN agent. QLearning agent(std::move(config), std::move(model), std::move(policy), - std::move(replayMethod)); + std::move(replayMethod)); arma::running_stat averageReturn; size_t episodes = 0; @@ -133,7 +135,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN) // Set up the DQN agent. QLearning agent(std::move(config), std::move(model), std::move(policy), - std::move(replayMethod)); + std::move(replayMethod)); arma::running_stat averageReturn; @@ -170,4 +172,70 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN) BOOST_REQUIRE(converged); } +//! Test DQN in Acrobat task. +BOOST_AUTO_TEST_CASE(AcrobatWithDQN) +{ + // Set up the network. + FFN, GaussianInitialization> model(MeanSquaredError<>(), + GaussianInitialization(0, 0.001)); + model.Add>(4, 64); + model.Add>(); + model.Add>(64, 32); + model.Add>(); + model.Add>(32, 3); + + // Set up the policy and replay method. + GreedyPolicy policy(1.0, 1000, 0.1); + RandomReplay replayMethod(20, 10000); + + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.99; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 400; + + // Set up DQN agent. + QLearning + agent(std::move(config), std::move(model), std::move(policy), + std::move(replayMethod)); + + arma::running_stat averageReturn; + size_t episodes = 0; + bool converged = true; + while (true) + { + double episodeReturn = agent.Episode(); + averageReturn(episodeReturn); + episodes += 1; + + if (episodes > 1000) + { + Log::Debug << "Acrobat with DQN failed." << std::endl; + converged = false; + break; + } + + /** + * I am using a thresold of -380 to check convegence. + */ + Log::Debug << "Average return: " << averageReturn.mean() + << " Episode return: " << episodeReturn << std::endl; + if (averageReturn.mean() > -380.00) + { + agent.Deterministic() = true; + arma::running_stat testReturn; + for (size_t i = 0; i < 20; ++i) + testReturn(agent.Episode()); + + Log::Debug << "Average return in deterministic test: " + << testReturn.mean() << std::endl; + break; + } + } + + BOOST_REQUIRE(converged); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 3daa866225..46e528474e 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -12,9 +12,11 @@ #include #include +#include #include #include #include +#include #include #include "test_tools.hpp" @@ -24,6 +26,7 @@ using namespace mlpack; using namespace mlpack::ann; using namespace mlpack::optimization; +using namespace mlpack::math; BOOST_AUTO_TEST_SUITE(RecurrentNetworkTest); @@ -1085,4 +1088,139 @@ BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest) ReberGrammarTestCustomNetwork(16, true); } +/** + * @brief Generates noisy sine wave and outputs the data and the labels that + * can be used directly for training and testing with RNN. + * + * @param data The data points as output + * @param labels The expected values as output + * @param rho The size of the sequence of each data point + * @param outputSteps How many output steps to consider for every rho inputs + * @param dataPoints The number of generated data points. The actual generated + * data points may be more than this to adjust to the outputSteps. But at + * the minimum these many data points will be generated. + * @param gain The gain on the amplitude + * @param freq The frquency of the sine wave + * @param phase The phase shift if any + * @param noisePercent The percent noise to induce + * @param numCycles How many full size wave cycles required. All the data + * points will be fit into these cycles. + * @param normalize Whether to normalise the data. This may be required for some + * layers like LSTM. Default is true. + */ +void GenerateNoisySinRNN(arma::cube& data, + arma::cube& labels, + size_t rho, + size_t outputSteps = 1, + const int dataPoints = 100, + const double gain = 1.0, + const int freq = 10, + const double phase = 0, + const int noisePercent = 20, + const double numCycles = 6.0, + const bool normalize = true) +{ + int points = dataPoints; + int r = dataPoints % rho; + + if (r == 0) + { + points += outputSteps; + } + else + { + points += rho - r + outputSteps; + } + + arma::colvec x(points); + int i = 0; + double interval = numCycles / freq / points; + + x.for_each([&i, gain, freq, phase, noisePercent, interval] + (arma::colvec::elem_type& val) { + double t = interval * (i++); + val = gain * ::sin(2 * M_PI * freq * t + phase) + + (noisePercent * gain / 100 * Random(0.0, 0.1)); + }); + + arma::colvec y = x; + if (normalize) + y = arma::normalise(x); + + // Now break this into columns of rho size slices. + size_t numColumns = y.n_elem / rho; + data = arma::cube(1, numColumns, rho); + labels = arma::cube(outputSteps, numColumns, 1); + + for (size_t i = 0; i < numColumns; ++i) + { + data.tube(0, i) = y.rows(i * rho, i * rho + rho - 1); + labels.subcube(0, i, 0, outputSteps - 1, i, 0) = + y.rows(i * rho + rho, i * rho + rho + outputSteps - 1); + } +} + +/** + * @brief RNNSineTest Test a simple RNN using noisy sine. Use single output + * for multiple inputs. + * @param hiddenUnits No of units in the hiddenlayer. + * @param rho The input sequence length. + * @param numEpochs The number of epochs to run. + * @return The mean squared error of the prediction. + */ +double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100) +{ + RNN > net(rho, true); + net.Add >(1, hiddenUnits); + net.Add >(hiddenUnits, hiddenUnits); + net.Add >(hiddenUnits, 1); + + RMSProp opt(0.005, 100, 0.9, 1e-08, 50000, 1e-5); + + // Generate data + arma::cube data; + arma::cube labels; + GenerateNoisySinRNN(data, labels, rho, 1, 2000, 20.0, 200, 0.0, 45, 20); + + // Break into training and test sets. Simply split along columns. + size_t trainCols = data.n_cols * 0.8; // Take 20% out for testing. + size_t testCols = data.n_cols - trainCols; + arma::cube testData = data.subcube(0, data.n_cols - testCols, 0, + data.n_rows - 1, data.n_cols - 1, data.n_slices - 1); + arma::cube testLabels = labels.subcube(0, labels.n_cols - testCols, 0, + labels.n_rows - 1, labels.n_cols - 1, labels.n_slices - 1); + + for (size_t i = 0; i < numEpochs; ++i) + { + net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1, + data.n_slices - 1), labels.subcube(0, 0, 0, labels.n_rows - 1, + trainCols - 1, labels.n_slices - 1), opt); + } + // Well now it should be trained. Do the test here. + arma::cube prediction; + net.Predict(testData, prediction); + + // The prediction must really follow the test data. So convert both the test + // data and the pediction to vectors and compare the two. + arma::colvec testVector = arma::vectorise(testData); + arma::colvec predVector = arma::vectorise(prediction); + + // Adjust the vectors for comparison, as the prediction is one step ahead. + testVector = testVector.rows(1, testVector.n_rows - 1); + predVector = predVector.rows(0, predVector.n_rows - 2); + double error = std::sqrt(arma::sum(arma::square(testVector - predVector))) / + testVector.n_rows; + + return error; +} + +/** + * Test RNN using multiple timestep input and single output. + */ +BOOST_AUTO_TEST_CASE(MultiTimestepTest) +{ + double err = RNNSineTest(4, 10, 20); + BOOST_REQUIRE_LE(err, 1e-02); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 5ee77f3350..2d7026ac7c 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -25,6 +26,23 @@ using namespace mlpack::rl; BOOST_AUTO_TEST_SUITE(RLComponentsTest) +/** + * Constructs a Acrobat instance and check if the main rountine works as + * it should be. + */ +BOOST_AUTO_TEST_CASE(SimpleAcrobatTest) +{ + const Acrobat task = Acrobat(); + + Acrobat::State state = task.InitialSample(); + Acrobat::Action action = Acrobat::Action::negativeTorque; + double reward = task.Sample(state, action); + + BOOST_REQUIRE_EQUAL(reward, -1.0); + BOOST_REQUIRE(!task.IsTerminal(state)); + BOOST_REQUIRE_EQUAL(3, Acrobat::Action::size); +} + /** * Constructs a MountainCar instance and check if the main rountine works as * it should be.